From d65f2990ce4cce6ff59906389e32066afc838770 Mon Sep 17 00:00:00 2001 From: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> Date: Tue, 20 Jan 2026 11:18:35 +0530 Subject: [PATCH 01/95] Bump version to 2.5.3 and document features (#2110) --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03b6c3c90..14b77c719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -## 2.5.2 (Unreleased) +## 2.5.3 (Unreleased) +**Features** + +**Bug Fixes** + +## 2.5.2 (2026-01-19) **Features** - Add Build Support for arm 32bit.([PR #2068](https://github.com/Azure/azure-storage-fuse/pull/2068)) - Added Debian 13 (trixie) support for package installation.([#2064](https://github.com/Azure/azure-storage-fuse/issues/2064)) From e52f77ae5619f0ec9ac73e728eb3251539b72201 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 Jan 2026 17:21:16 +0530 Subject: [PATCH 02/95] Update Go version from 1.25.1 to 1.25.6 (#2115) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: souravgupta-msft <98318303+souravgupta-msft@users.noreply.github.com> --- .github/workflows/arm-ci.yml | 2 +- go.mod | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index f69f95a06..efa17e8d9 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -52,7 +52,7 @@ jobs: - name: Setup Go (cache modules) uses: actions/setup-go@v6 with: - go-version: '1.25.1' + go-version-file: 'go.mod' - name: Build ARM binary (cross-compile with sysroot includes) env: diff --git a/go.mod b/go.mod index 3b227ba9d..5549c94be 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/Azure/azure-storage-fuse/v2 -go 1.25.1 +go 1.25 + +toolchain go1.25.6 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 From 658ec3c26e873921a02269a849be749085a4791e Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 09:08:01 +0530 Subject: [PATCH 03/95] Disable FIO read tests for filecache mode and add custom 100GB direct I/O test (#2106) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: syeleti-msft <172502481+syeleti-msft@users.noreply.github.com> --- perf_testing/scripts/fio_bench.sh | 121 +++++++++++++++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/perf_testing/scripts/fio_bench.sh b/perf_testing/scripts/fio_bench.sh index 174e34e4c..c01f82e4f 100755 --- a/perf_testing/scripts/fio_bench.sh +++ b/perf_testing/scripts/fio_bench.sh @@ -183,6 +183,118 @@ run_test_suite() { done } +# -------------------------------------------------------------------------------------------------- +# Helper: Run custom filecache read test +run_filecache_read_test() { + echo "Running custom filecache read test..." + + local TEST_FILE="filecache_read_test_100gb.data" + local BLOCK_SIZE="1M" + + # Step 1: Mount and create the test file if it doesn't exist + mount_blobfuse + rm -rf "${MOUNT_DIR}/"* + + if [ ! -f "${MOUNT_DIR}/${TEST_FILE}" ]; then + echo "Creating 100GB test file: ${TEST_FILE}..." + echo "This may take some time..." + + # Create the file using dd (this will upload to blob storage in file_cache mode) + dd if=/dev/urandom of="${MOUNT_DIR}/${TEST_FILE}" bs=1M count=102400 status=progress + + echo "Test file created successfully." + + # Sync to ensure file is uploaded + sync + sleep 5 + fi + + # Verify file exists and get its size + if [ ! -f "${MOUNT_DIR}/${TEST_FILE}" ]; then + echo "Error: Test file does not exist" + exit 1 + fi + + local FILE_SIZE=$(stat -c%s "${MOUNT_DIR}/${TEST_FILE}") + echo "Test file size: $FILE_SIZE bytes" + + # Step 2: Unmount to clear any cached data + echo "Unmounting to clear cache..." + cleanup_mount + sleep 5 + + # Step 3: Clear kernel page cache + echo "Dropping kernel caches..." + sudo sh -c "echo 3 > /proc/sys/vm/drop_caches" + sleep 2 + + # Step 4: Mount again for the read test + echo "Remounting for read test..." + mount_blobfuse + + # Step 5: Run the read test with direct I/O + echo "Starting sequential read test with direct I/O (block size: ${BLOCK_SIZE})..." + + # Get initial network stats + local start_rx=$(cat /sys/class/net/$INTERFACE/statistics/rx_bytes) + local start_tx=$(cat /sys/class/net/$INTERFACE/statistics/tx_bytes) + local start_time=$(date +%s.%N) + + # Perform the read test using dd with direct I/O + # O_DIRECT flag ensures we bypass the OS page cache + # This includes the open time (when filecache downloads the file) + dd if="${MOUNT_DIR}/${TEST_FILE}" of=/dev/null bs=${BLOCK_SIZE} iflag=direct 2>&1 | tee "${OUTPUT_DIR}/dd_output.txt" + + # Get final stats + local end_time=$(date +%s.%N) + local end_rx=$(cat /sys/class/net/$INTERFACE/statistics/rx_bytes) + local end_tx=$(cat /sys/class/net/$INTERFACE/statistics/tx_bytes) + + # Calculate metrics + local duration=$(echo "$end_time - $start_time" | bc) + local rx_bytes=$((end_rx - start_rx)) + local tx_bytes=$((end_tx - start_tx)) + + # Calculate bandwidth in Mbps + local rx_mbps=$(echo "scale=4; ($rx_bytes * 8) / ($duration * 1000000)" | bc) + local tx_mbps=$(echo "scale=4; ($tx_bytes * 8) / ($duration * 1000000)" | bc) + + # Calculate throughput in MiB/s (includes open time) + local throughput_mibs=$(echo "scale=4; $FILE_SIZE / ($duration * 1024 * 1024)" | bc) + + echo "-------------------------------------" + echo "Test completed!" + echo "Duration: ${duration} seconds" + echo "Throughput: ${throughput_mibs} MiB/s (including file open time)" + echo + echo "Network Statistics:" + echo "Interface: $INTERFACE" + echo "Received (RX): $rx_bytes bytes (${rx_mbps} Mbps)" + echo "Transmitted (TX): $tx_bytes bytes (${tx_mbps} Mbps)" + echo "-------------------------------------" + + # Generate JSON output for bandwidth results + cat > "${OUTPUT_DIR}/filecache_sequential_read_bandwidth_summary.json" < "${OUTPUT_DIR}/filecache_sequential_read_latency_summary.json" < Date: Mon, 9 Feb 2026 11:08:34 +0530 Subject: [PATCH 04/95] Bump golang.org/x/sys from 0.40.0 to 0.41.0 (#2122) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5549c94be..39dfe1b29 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.47.0 // indirect golang.org/x/net v0.49.0 // indirect - golang.org/x/sys v0.40.0 + golang.org/x/sys v0.41.0 golang.org/x/text v0.33.0 // indirect ) diff --git a/go.sum b/go.sum index bedd4f2f5..8e26bd483 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 8cfdd0212d34abc1e48e7c2b8bbf92c840c2d4ef Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Mon, 9 Feb 2026 11:37:45 +0530 Subject: [PATCH 05/95] Adding rate limit functionality for ingress bandwidth and operations per second (#2093) --- CHANGELOG.md | 1 + NOTICE | 41 ++++++ component/azstorage/azstorage.go | 6 + component/azstorage/config.go | 17 ++- component/azstorage/config_test.go | 34 +++++ component/azstorage/connection.go | 4 + component/azstorage/policies.go | 103 ++++++++++++++ component/azstorage/policies_test.go | 198 +++++++++++++++++++++++++++ component/azstorage/utils.go | 65 ++++++++- component/azstorage/utils_test.go | 30 ++++ go.mod | 1 + go.sum | 2 + setup/advancedConfig.yaml | 2 + 13 files changed, 499 insertions(+), 5 deletions(-) create mode 100644 component/azstorage/policies_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b77c719..33506c72a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## 2.5.3 (Unreleased) **Features** +- Add rate limit functionality for ingress bandwidth (bytes downloaded per second) and operations per second ([PR #2093](https://github.com/Azure/azure-storage-fuse/pull/2093)) **Bug Fixes** diff --git a/NOTICE b/NOTICE index fab5a4c6c..828f5e7a3 100644 --- a/NOTICE +++ b/NOTICE @@ -4559,4 +4559,45 @@ Apache License + + + + +**************************************************************************** + +============================================================================ +>>> golang.org/x/time +============================================================================== + +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + + --------------------- END OF THIRD PARTY NOTICE -------------------------------- diff --git a/component/azstorage/azstorage.go b/component/azstorage/azstorage.go index db19803b1..f4c6a2b1b 100644 --- a/component/azstorage/azstorage.go +++ b/component/azstorage/azstorage.go @@ -693,6 +693,12 @@ func init() { blobFilter := config.AddStringFlag("filter", "", "Filter string to match blobs. For details refer [https://github.com/Azure/azure-storage-fuse?tab=readme-ov-file#blob-filter]") config.BindPFlag(compName+".filter", blobFilter) + capMbpsRead := config.AddInt64Flag("cap-mbps-read", -1, "Limit the throughput of downloads from your storage account. Value measured in megabits per second. Default is -1 (no limit)") + config.BindPFlag(compName+".cap-mbps-read", capMbpsRead) + + capIOps := config.AddInt64Flag("cap-iops", -1, "Limit the total storage operations per second. Default is -1 (no limit)") + config.BindPFlag(compName+".cap-iops", capIOps) + config.RegisterFlagCompletionFunc("container-name", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return nil, cobra.ShellCompDirectiveNoFileComp }) diff --git a/component/azstorage/config.go b/component/azstorage/config.go index 3b783b7b0..b04a50b13 100644 --- a/component/azstorage/config.go +++ b/component/azstorage/config.go @@ -195,6 +195,8 @@ type AzStorageOptions struct { PreserveACL bool `config:"preserve-acl" yaml:"preserve-acl"` Filter string `config:"filter" yaml:"filter"` UserAssertion string `config:"user-assertion" yaml:"user-assertions"` + CapMbpsRead int64 `config:"cap-mbps-read" yaml:"cap-mbps-read"` + CapIOps int64 `config:"cap-iops" yaml:"cap-iops"` // v1 support UseAdls bool `config:"use-adls" yaml:"-"` @@ -537,7 +539,8 @@ func ParseAndValidateConfig(az *AzStorage, opt AzStorageOptions) error { log.Crit("ParseAndValidateConfig : Retry Config: retry-count %d, max-timeout %d, backoff-time %d, max-delay %d, preserve-acl: %v", az.stConfig.maxRetries, az.stConfig.maxTimeout, az.stConfig.backoffTime, az.stConfig.maxRetryDelay, az.stConfig.preserveACL) - log.Crit("ParseAndValidateConfig : Telemetry : %s, honour-ACL %v", az.stConfig.telemetry, az.stConfig.honourACL) + log.Crit("ParseAndValidateConfig : Telemetry : %s, honour-ACL %v, cap-mbps-read %d, cap-iops %d", + az.stConfig.telemetry, az.stConfig.honourACL, az.stConfig.capMbpsRead, az.stConfig.capIOps) return nil } @@ -630,5 +633,17 @@ func ParseAndReadDynamicConfig(az *AzStorage, opt AzStorageOptions, reload bool) } } + // Rate limiting, default is no limit + az.stConfig.capMbpsRead = -1 + az.stConfig.capIOps = -1 + + if opt.CapMbpsRead > 0 { + az.stConfig.capMbpsRead = opt.CapMbpsRead + } + + if opt.CapIOps > 0 { + az.stConfig.capIOps = opt.CapIOps + } + return nil } diff --git a/component/azstorage/config_test.go b/component/azstorage/config_test.go index 540b68d55..576582da1 100644 --- a/component/azstorage/config_test.go +++ b/component/azstorage/config_test.go @@ -437,6 +437,40 @@ func (s *configTestSuite) TestSASRefresh() { assert.NoError(err) } +func (s *configTestSuite) TestRateLimitConfig() { + defer config.ResetConfig() + assert := assert.New(s.T()) + az := &AzStorage{} + opt := AzStorageOptions{} + opt.AccountName = "abcd" + opt.Container = "abcd" + opt.AuthMode = "key" + opt.AccountKey = "abcd" + + // Test default values (no limit) + err := ParseAndReadDynamicConfig(az, opt, false) + assert.NoError(err) + assert.Equal(int64(-1), az.stConfig.capMbpsRead) + assert.Equal(int64(-1), az.stConfig.capIOps) + + // Test setting limits + opt.CapMbpsRead = 100 + opt.CapIOps = 10 + err = ParseAndReadDynamicConfig(az, opt, false) + assert.NoError(err) + assert.Equal(int64(100), az.stConfig.capMbpsRead) + assert.Equal(int64(10), az.stConfig.capIOps) + + // Test setting only one limit + opt.CapMbpsRead = 200 + opt.CapIOps = 0 // reset to no limit + + err = ParseAndReadDynamicConfig(az, opt, false) + assert.NoError(err) + assert.Equal(int64(200), az.stConfig.capMbpsRead) + assert.Equal(int64(-1), az.stConfig.capIOps) +} + func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(configTestSuite)) } diff --git a/component/azstorage/connection.go b/component/azstorage/connection.go index 85f088f96..d880c0967 100644 --- a/component/azstorage/connection.go +++ b/component/azstorage/connection.go @@ -85,6 +85,10 @@ type AzStorageConfig struct { // Blob filters filter *blobfilter.BlobFilter + + // Rate limiting + capMbpsRead int64 + capIOps int64 } type AzStorageConnection struct { diff --git a/component/azstorage/policies.go b/component/azstorage/policies.go index 121e8e5ae..8ff74cec8 100644 --- a/component/azstorage/policies.go +++ b/component/azstorage/policies.go @@ -35,10 +35,15 @@ package azstorage import ( "fmt" + "math" "net/http" + "strings" + + "golang.org/x/time/rate" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-storage-fuse/v2/common" + "github.com/Azure/azure-storage-fuse/v2/common/log" ) // blobfuseTelemetryPolicy is a custom pipeline policy to prepend the blobfuse user agent string to the one coming from SDK. @@ -79,3 +84,101 @@ func (r *serviceVersionPolicy) Do(req *policy.Request) (*http.Response, error) { req.Raw().Header["x-ms-version"] = []string{r.serviceApiVersion} return req.Next() } + +// --------------------------------------------------------------------------------------------------------------------------------------------------- +// Policy to limit the rate of requests +type rateLimitingPolicy struct { + ingressBandwidthLimiter *rate.Limiter + opsLimiter *rate.Limiter +} + +func newRateLimitingPolicy(readBytesPerSec int64, opsPerSec int64) policy.Policy { + p := &rateLimitingPolicy{} + + // Use 10 second window for burst size calculation for rate limiter. + // This allows for short bursts while still enforcing the average rate over a reasonable time period. + // A larger window size would allow larger bursts but would be less effective at limiting short term spikes. + // A smaller window size would limit bursts more but could lead to underutilization of available bandwidth/ops. + // 10 seconds is a reasonable compromise between these factors. + // Note: The burst size is the maximum number of tokens that can be accumulated in the bucket. + // Therefore, a larger burst size allows for larger bursts of traffic, + // but does not affect the average rate over time. + // Users can adjust the bytesPerSec and opsPerSec values to fine-tune the rate limiting behavior as needed. + // For example, setting a higher bytesPerSec value will allow for higher average bandwidth, + // while setting a lower opsPerSec value will limit the number of operations per second. + const windowSize = 10 + + if readBytesPerSec > 0 { + ingressBandwidthBurstSize := readBytesPerSec * int64(windowSize) + burst := int(ingressBandwidthBurstSize) + // On 32-bit systems, int is 32-bit. If bandwidthBurstSize > MaxInt, we need to clamp it. + // math.MaxInt is platform dependent. + if ingressBandwidthBurstSize > int64(math.MaxInt) { + burst = math.MaxInt + } + + p.ingressBandwidthLimiter = rate.NewLimiter(rate.Limit(readBytesPerSec), burst) + log.Info("RateLimitingPolicy : Bandwidth limit set to %d bytes/sec with burst size of %d bytes", + readBytesPerSec, burst) + } + + if opsPerSec > 0 { + opsBurstSize := opsPerSec * int64(windowSize) + burst := int(opsBurstSize) + if opsBurstSize > int64(math.MaxInt) { + burst = math.MaxInt + } + + p.opsLimiter = rate.NewLimiter(rate.Limit(opsPerSec), burst) + log.Info("RateLimitingPolicy : Ops limit set to %d ops/sec with burst size of %d ops", + opsPerSec, burst) + } + + return p +} + +func (p *rateLimitingPolicy) Do(req *policy.Request) (*http.Response, error) { + ctx := req.Raw().Context() + + // Limit operations per second + if p.opsLimiter != nil { + // Wait for 1 token + err := p.opsLimiter.Wait(ctx) + if err != nil { + log.Err("RateLimitingPolicy : Ops limit wait failed [%s]", err.Error()) + return nil, err + } + } + + // Limit ingress bandwidth for blob downloads (Azure egress: data leaving Azure Storage). + // This policy intentionally applies only to GET requests, which represent download operations. + if p.ingressBandwidthLimiter != nil && req.Raw().Method == http.MethodGet { + // Check for x-ms-range header + // We are not using req.Raw().Header.Get() as it canonicalizes the header name. + // Whereas SDK stores the header in the request is stored in lower case. + // So we directly access the header map with lower case key. + // NOTE: using strings.ToLower to ignore the lint error regarding canonicalized headers. + rangeHeader := req.Raw().Header[strings.ToLower(X_Ms_Range)] + if len(rangeHeader) == 0 { + rangeHeader = req.Raw().Header[RangeHeader] + } + + if len(rangeHeader) > 0 { + size, err := parseRangeHeader(rangeHeader[0]) + if err == nil && size > 0 { + // Wait for tokens equal to size. + // NOTE: range size is guaranteed to be within int range by parseRangeHeader. + err := p.ingressBandwidthLimiter.WaitN(ctx, int(size)) + if err != nil { + log.Err("RateLimitingPolicy : Bandwidth limit wait failed [%s]", err.Error()) + return nil, err + } + } else if err != nil { + log.Err("RateLimitingPolicy : Failed to parse Range header %s: [%s]", rangeHeader[0], err.Error()) + return nil, err + } + } + } + + return req.Next() +} diff --git a/component/azstorage/policies_test.go b/component/azstorage/policies_test.go new file mode 100644 index 000000000..04240a9f2 --- /dev/null +++ b/component/azstorage/policies_test.go @@ -0,0 +1,198 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2026 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package azstorage + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type policiesTestSuite struct { + suite.Suite +} + +type mockTransport struct{} + +func (m *mockTransport) Do(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200}, nil +} + +func (s *policiesTestSuite) TestRateLimitingPolicy_OpsLimit() { + assert := assert.New(s.T()) + + // Limit to 1 op/sec. Burst will be 1 * 10 = 10 ops. + p := newRateLimitingPolicy(-1, 1) + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{ + PerRetry: []policy.Policy{p}, + }, &policy.ClientOptions{Transport: &mockTransport{}}) + + req, err := runtime.NewRequest(context.Background(), http.MethodGet, "http://localhost") + assert.NoError(err) + + // Consume burst + for i := 0; i < 10; i++ { + _, err := pipeline.Do(req) + assert.NoError(err) + } + + // Next request should be delayed by ~1 sec + start := time.Now() + _, err = pipeline.Do(req) + assert.NoError(err) + duration := time.Since(start) + + // It should take at least 1 second (minus some tolerance) + assert.GreaterOrEqual(duration, 900*time.Millisecond, "Expected delay of ~1s, got %v", duration) +} + +func (s *policiesTestSuite) TestRateLimitingPolicy_BandwidthLimit() { + assert := assert.New(s.T()) + + // Limit 100 bytes/sec. Burst will be 100 * 10 = 1000 bytes. + p := newRateLimitingPolicy(100, -1) + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{ + PerRetry: []policy.Policy{p}, + }, &policy.ClientOptions{Transport: &mockTransport{}}) + + req, err := runtime.NewRequest(context.Background(), http.MethodGet, "http://localhost") + assert.NoError(err) + + req.Raw().Header["Range"] = []string{"bytes=0-99"} // 100 bytes + + // Consume burst (10 requests of 100 bytes = 1000 bytes) + for i := 0; i < 10; i++ { + _, err := pipeline.Do(req) + assert.NoError(err) + } + + // Next request of 100 bytes should be delayed by ~1 sec + start := time.Now() + _, err = pipeline.Do(req) + assert.NoError(err) + duration := time.Since(start) + + assert.GreaterOrEqual(duration, 900*time.Millisecond, "Expected delay of ~1s, got %v", duration) +} + +func (s *policiesTestSuite) TestRateLimitingPolicy_NoLimit() { + assert := assert.New(s.T()) + + p := newRateLimitingPolicy(-1, -1) + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{ + PerRetry: []policy.Policy{p}, + }, &policy.ClientOptions{Transport: &mockTransport{}}) + + req, _ := runtime.NewRequest(context.Background(), http.MethodGet, "http://localhost") + req.Raw().Header["Range"] = []string{"bytes=0-99"} + + start := time.Now() + for i := 0; i < 20; i++ { + _, err := pipeline.Do(req) + assert.NoError(err) + } + duration := time.Since(start) + + // Should be very fast + assert.Less(duration, 100*time.Millisecond, "Expected fast execution, got %v", duration) +} + +func (s *policiesTestSuite) TestRateLimitingPolicy_BandwidthLimit_XMsRange() { + assert := assert.New(s.T()) + + // Limit 100 bytes/sec. Burst 1000 bytes. + p := newRateLimitingPolicy(100, -1) + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{ + PerRetry: []policy.Policy{p}, + }, &policy.ClientOptions{Transport: &mockTransport{}}) + + req, _ := runtime.NewRequest(context.Background(), http.MethodGet, "http://localhost") + req.Raw().Header["x-ms-range"] = []string{"bytes=0-99"} // 100 bytes + + // Consume burst + for i := 0; i < 10; i++ { + _, err := pipeline.Do(req) + assert.NoError(err) + } + + // Next request should be delayed + start := time.Now() + _, err := pipeline.Do(req) + assert.NoError(err) + duration := time.Since(start) + + assert.GreaterOrEqual(duration, 900*time.Millisecond, "Expected delay of ~1s, got %v", duration) +} + +func (s *policiesTestSuite) TestRateLimitingPolicy_BandwidthLimit_SkipNonGet() { + assert := assert.New(s.T()) + + // Limit 100 bytes/sec. burst 1000. + p := newRateLimitingPolicy(100, -1) + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{ + PerRetry: []policy.Policy{p}, + }, &policy.ClientOptions{Transport: &mockTransport{}}) + + // Create requests for checkable methods + methods := []string{http.MethodPut, http.MethodPost, http.MethodDelete, http.MethodHead} + + for _, method := range methods { + req, _ := runtime.NewRequest(context.Background(), method, "http://localhost") + // Even if we have a range header that implies a large payload + // the policy should ignore it because it's not GET. + req.Raw().Header["Range"] = []string{"bytes=0-999"} // 1000 bytes + + start := time.Now() + // Execute multiple times - if limited, this would take ~10 seconds (1000 bytes * 10 / 100 bytes/sec) + // But since we are skipping non-GET, it should be instant. + for i := 0; i < 11; i++ { + _, err := pipeline.Do(req) + assert.NoError(err) + } + duration := time.Since(start) + + // Each request should be effectively instant, so total should be very fast + assert.Less(duration, 100*time.Millisecond, "Expected fast execution for method %s, got %v", method, duration) + } +} + +func TestPoliciesSuite(t *testing.T) { + suite.Run(t, new(policiesTestSuite)) +} diff --git a/component/azstorage/utils.go b/component/azstorage/utils.go index bea566955..31479cdcc 100644 --- a/component/azstorage/utils.go +++ b/component/azstorage/utils.go @@ -43,6 +43,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "time" @@ -80,6 +81,9 @@ const ( DisableKeepAlives bool = false DisableCompression bool = false MaxResponseHeaderBytes int64 = 0 + + X_Ms_Range string = "x-ms-range" + RangeHeader string = "Range" ) // getAzStorageClientOptions : Create client options based on the config @@ -113,11 +117,19 @@ func getAzStorageClientOptions(conf *AzStorageConfig) (azcore.ClientOptions, err perCallPolicies = append(perCallPolicies, newServiceVersionPolicy(serviceApiVersion)) } + perRetryPolicies := []policy.Policy{} + if conf.capMbpsRead > 0 || conf.capIOps > 0 { + // Convert Mbps to Bytes/sec: 1 Mbps = (1024* 1024) / 8 = 131072 Bytes/sec + bytesPerSec := conf.capMbpsRead * 131072 + perRetryPolicies = append(perRetryPolicies, newRateLimitingPolicy(bytesPerSec, conf.capIOps)) + } + return azcore.ClientOptions{ - Retry: retryOptions, - Logging: logOptions, - PerCallPolicies: perCallPolicies, - Transport: transportOptions, + Retry: retryOptions, + Logging: logOptions, + PerCallPolicies: perCallPolicies, + PerRetryPolicies: perRetryPolicies, + Transport: transportOptions, }, err } @@ -583,6 +595,51 @@ func sanitizeEtag(ETag *azcore.ETag) string { return "" } +// parseRangeHeader parses the x-ms-range header and returns the size of the range requested. +// Examples of x-ms-range header: +// +// bytes=0-499 --> returns 500 +// bytes=500-999 --> returns 500 +// bytes=500- --> returns error (open ended range) +// bytes=-500 --> returns error +// bytes=1000-500 --> returns error (invalid range) +func parseRangeHeader(rangeHeader string) (int64, error) { + if rangeHeader == "" { + return 0, fmt.Errorf("empty x-ms-range header") + } + + if !strings.HasPrefix(rangeHeader, "bytes=") { + return 0, fmt.Errorf("invalid x-ms-range header format %s", rangeHeader) + } + + parts := strings.Split(strings.TrimPrefix(rangeHeader, "bytes="), "-") + if len(parts) != 2 { + return 0, fmt.Errorf("invalid x-ms-range header format %s", rangeHeader) + } + + start, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return 0, err + } + + // Open ended range + if parts[1] == "" { + return 0, fmt.Errorf("invalid x-ms-range header format %s, open ended range not supported", rangeHeader) + } + + end, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return 0, err + } + + // Invalid range + if end < start { + return 0, fmt.Errorf("invalid range %s", rangeHeader) + } + + return end - start + 1, nil +} + // func parseBlobTags(tags *container.BlobTags) map[string]string { // if tags == nil { diff --git a/component/azstorage/utils_test.go b/component/azstorage/utils_test.go index 54b4ac3b1..9123b71b0 100644 --- a/component/azstorage/utils_test.go +++ b/component/azstorage/utils_test.go @@ -512,6 +512,36 @@ func (s *utilsTestSuite) TestRemovePrefixPath() { } } +func (s *utilsTestSuite) TestParseRangeHeader() { + assert := assert.New(s.T()) + + tests := []struct { + header string + expected int64 + hasError bool + }{ + {"bytes=0-100", 101, false}, + {"bytes=100-200", 101, false}, + {"bytes=0-0", 1, false}, + {"bytes=0-", 0, true}, // open ended range not supported + {"", 0, true}, + {"invalid", 0, true}, + {"bytes=abc-def", 0, true}, + {"bytes=100-50", 0, true}, // invalid range + } + + for _, test := range tests { + size, err := parseRangeHeader(test.header) + + if test.hasError { + assert.Error(err) + } else { + assert.NoError(err) + assert.Equal(test.expected, size) + } + } +} + func TestUtilsTestSuite(t *testing.T) { suite.Run(t, new(utilsTestSuite)) } diff --git a/go.mod b/go.mod index 39dfe1b29..c74187f4b 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/vibhansa-msft/blobfilter v0.0.0-20250115104552-d9d40722be3e github.com/vibhansa-msft/tlru v0.0.0-20240410102558-9e708419e21f go.uber.org/atomic v1.11.0 + golang.org/x/time v0.14.0 gopkg.in/ini.v1 v1.67.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 8e26bd483..2ac17b135 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= diff --git a/setup/advancedConfig.yaml b/setup/advancedConfig.yaml index 3380639ef..efe1342b5 100644 --- a/setup/advancedConfig.yaml +++ b/setup/advancedConfig.yaml @@ -164,6 +164,8 @@ azstorage: cpk-encryption-key: cpk-encryption-key-sha256: preserve-acl: true|false + cap-mbps-read: + cap-iops: # Mount all configuration mountall: From ef8e02089302fe23012c503a666b5b4a061d5356 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 18 Feb 2026 19:44:32 +0530 Subject: [PATCH 06/95] Fix QF1012 staticcheck warnings: replace WriteString(Sprintf) with Fprintf (#2131) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: syeleti-msft <172502481+syeleti-msft@users.noreply.github.com> --- .gitignore | 3 +++ cmd/gen-config.go | 2 +- component/attr_cache/attr_cache.go | 4 ++-- component/block_cache/block_cache.go | 16 ++++++++-------- component/file_cache/file_cache.go | 6 +++--- component/libfuse/libfuse.go | 8 ++++---- 6 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index daa0662aa..4f2bea89e 100755 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ component/azstorage/logfile.txt logfile.txt **/logfile.txt **/logfile.txt.* +# Test trace files +*/_tmp_*.trace +_tmp_*.trace # Emacs backup and auto-save files *~ \#*\# diff --git a/cmd/gen-config.go b/cmd/gen-config.go index e8987c86f..bf9db476e 100644 --- a/cmd/gen-config.go +++ b/cmd/gen-config.go @@ -113,7 +113,7 @@ var generatedConfig = &cobra.Command{ sb.WriteString("\ncomponents:\n") for _, component := range pipeline { - sb.WriteString(fmt.Sprintf(" - %s\n", component)) + fmt.Fprintf(&sb, " - %s\n", component) } for _, component := range pipeline { diff --git a/component/attr_cache/attr_cache.go b/component/attr_cache/attr_cache.go index e1608108d..b3c325c1b 100644 --- a/component/attr_cache/attr_cache.go +++ b/component/attr_cache/attr_cache.go @@ -137,8 +137,8 @@ func (ac *AttrCache) GenConfig() string { log.Info("AttrCache::Configure : config generation started") var sb strings.Builder - sb.WriteString(fmt.Sprintf("\n%s:", ac.Name())) - sb.WriteString(fmt.Sprintf("\n timeout-sec: %v", defaultAttrCacheTimeout)) + fmt.Fprintf(&sb, "\n%s:", ac.Name()) + fmt.Fprintf(&sb, "\n timeout-sec: %v", defaultAttrCacheTimeout) return sb.String() } diff --git a/component/block_cache/block_cache.go b/component/block_cache/block_cache.go index 4a4f40f0f..da7ff5224 100755 --- a/component/block_cache/block_cache.go +++ b/component/block_cache/block_cache.go @@ -199,19 +199,19 @@ func (bc *BlockCache) GenConfig() string { } var sb strings.Builder - sb.WriteString(fmt.Sprintf("\n%s:", bc.Name())) + fmt.Fprintf(&sb, "\n%s:", bc.Name()) - sb.WriteString(fmt.Sprintf("\n block-size-mb: %v", defaultBlockSize)) - sb.WriteString(fmt.Sprintf("\n mem-size-mb: %v", memSize)) - sb.WriteString(fmt.Sprintf("\n prefetch: %v", prefetch)) - sb.WriteString(fmt.Sprintf("\n parallelism: %v", uint32(3*runtime.NumCPU()))) + fmt.Fprintf(&sb, "\n block-size-mb: %v", defaultBlockSize) + fmt.Fprintf(&sb, "\n mem-size-mb: %v", memSize) + fmt.Fprintf(&sb, "\n prefetch: %v", prefetch) + fmt.Fprintf(&sb, "\n parallelism: %v", uint32(3*runtime.NumCPU())) var tmpPath = "" _ = config.UnmarshalKey("tmp-path", &tmpPath) if tmpPath != "" { - sb.WriteString(fmt.Sprintf("\n path: %v", tmpPath)) - sb.WriteString(fmt.Sprintf("\n disk-size-mb: %v", bc.getDefaultDiskSize(tmpPath))) - sb.WriteString(fmt.Sprintf("\n disk-timeout-sec: %v", defaultTimeout)) + fmt.Fprintf(&sb, "\n path: %v", tmpPath) + fmt.Fprintf(&sb, "\n disk-size-mb: %v", bc.getDefaultDiskSize(tmpPath)) + fmt.Fprintf(&sb, "\n disk-timeout-sec: %v", defaultTimeout) } return sb.String() diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index e3faf0143..2dab61a44 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -191,7 +191,7 @@ func (fc *FileCache) GenConfig() string { log.Info("FileCache::Configure : config generation started") var sb strings.Builder - sb.WriteString(fmt.Sprintf("\n%s:", fc.Name())) + fmt.Fprintf(&sb, "\n%s:", fc.Name()) tmpPath := "" _ = config.UnmarshalKey("tmp-path", &tmpPath) @@ -204,8 +204,8 @@ func (fc *FileCache) GenConfig() string { timeout = 0 } - sb.WriteString(fmt.Sprintf("\n path: %v", common.ExpandPath(tmpPath))) - sb.WriteString(fmt.Sprintf("\n timeout-sec: %v", timeout)) + fmt.Fprintf(&sb, "\n path: %v", common.ExpandPath(tmpPath)) + fmt.Fprintf(&sb, "\n timeout-sec: %v", timeout) return sb.String() } diff --git a/component/libfuse/libfuse.go b/component/libfuse/libfuse.go index 9e76623dd..9f14e4fc6 100644 --- a/component/libfuse/libfuse.go +++ b/component/libfuse/libfuse.go @@ -268,7 +268,7 @@ func (lf *Libfuse) GenConfig() string { _ = config.UnmarshalKey("direct-io", &directIO) var sb strings.Builder - sb.WriteString(fmt.Sprintf("\n%s:", lf.Name())) + fmt.Fprintf(&sb, "\n%s:", lf.Name()) timeout := defaultEntryExpiration if directIO { @@ -276,9 +276,9 @@ func (lf *Libfuse) GenConfig() string { sb.WriteString("\n direct-io: true") } - sb.WriteString(fmt.Sprintf("\n attribute-expiration-sec: %v", timeout)) - sb.WriteString(fmt.Sprintf("\n entry-expiration-sec: %v", timeout)) - sb.WriteString(fmt.Sprintf("\n negative-entry-expiration-sec: %v", timeout)) + fmt.Fprintf(&sb, "\n attribute-expiration-sec: %v", timeout) + fmt.Fprintf(&sb, "\n entry-expiration-sec: %v", timeout) + fmt.Fprintf(&sb, "\n negative-entry-expiration-sec: %v", timeout) return sb.String() } From 5a624fde85cb5ceea48a2fe9d0c114bc2a3eb91e Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Wed, 18 Feb 2026 21:51:06 +0530 Subject: [PATCH 07/95] Resolve panic in unit tests in policies_test.go (#2129) --- component/azstorage/policies_test.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/component/azstorage/policies_test.go b/component/azstorage/policies_test.go index 04240a9f2..94abd214a 100644 --- a/component/azstorage/policies_test.go +++ b/component/azstorage/policies_test.go @@ -41,18 +41,32 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-storage-fuse/v2/common" + "github.com/Azure/azure-storage-fuse/v2/common/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) +type mockTransport struct{} + +func (m *mockTransport) Do(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200}, nil +} + type policiesTestSuite struct { suite.Suite } -type mockTransport struct{} +func (s *policiesTestSuite) SetupTest() { + // Initialize the logger + err := log.SetDefaultLogger("silent", common.LogConfig{Level: common.ELogLevel.LOG_DEBUG()}) + if err != nil { + panic("Unable to set silent logger as default.") + } +} -func (m *mockTransport) Do(req *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 200}, nil +func (s *policiesTestSuite) TearDownTest() { + _ = log.Destroy() } func (s *policiesTestSuite) TestRateLimitingPolicy_OpsLimit() { From 326642f42d1d2f83e5815bc7b3e2ae22e373ea97 Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Thu, 19 Feb 2026 08:18:48 +0530 Subject: [PATCH 08/95] Update dependencies (#2132) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index c74187f4b..5bf9ae6d2 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect @@ -50,8 +50,8 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.47.0 // indirect - golang.org/x/net v0.49.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.50.0 // indirect golang.org/x/sys v0.41.0 - golang.org/x/text v0.33.0 // indirect + golang.org/x/text v0.34.0 // indirect ) diff --git a/go.sum b/go.sum index 2ac17b135..a7b06bbde 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -107,14 +107,14 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -128,8 +128,8 @@ golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 7ac0ad91d5b6692f2a85cb183cc3fe09d3b79ffc Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Thu, 19 Feb 2026 11:53:01 +0530 Subject: [PATCH 09/95] Resolve panic while reading an archived file using filecache (#2127) Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: syeleti-msft <172502481+syeleti-msft@users.noreply.github.com> --- CHANGELOG.md | 1 + component/file_cache/file_cache.go | 6 +- component/file_cache/file_cache_test.go | 93 +++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33506c72a..7add33075 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - Add rate limit functionality for ingress bandwidth (bytes downloaded per second) and operations per second ([PR #2093](https://github.com/Azure/azure-storage-fuse/pull/2093)) **Bug Fixes** +- Fix panic while reading an archived blob using file-cache ([PR #2127](https://github.com/Azure/azure-storage-fuse/pull/2127)] ## 2.5.2 (2026-01-19) **Features** diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index 2dab61a44..0eb87b880 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -993,9 +993,9 @@ func (fc *FileCache) OpenFile(options internal.OpenFileOptions) (*handlemap.Hand // File was created locally and now download has failed so we need to delete it back from local cache log.Err("FileCache::OpenFile : error downloading file from storage %s [%s]", options.Name, err.Error()) _ = f.Close() - err = os.Remove(localPath) - if err != nil { - log.Err("FileCache::OpenFile : Failed to remove file %s [%s]", localPath, err.Error()) + err2 := os.Remove(localPath) + if err2 != nil { + log.Err("FileCache::OpenFile : Failed to remove file %s [%s]", localPath, err2.Error()) } return nil, err } diff --git a/component/file_cache/file_cache_test.go b/component/file_cache/file_cache_test.go index 79d4d6601..4df36aefc 100644 --- a/component/file_cache/file_cache_test.go +++ b/component/file_cache/file_cache_test.go @@ -56,6 +56,7 @@ import ( "github.com/Azure/azure-storage-fuse/v2/internal" "github.com/Azure/azure-storage-fuse/v2/internal/handlemap" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -2150,6 +2151,98 @@ func (suite *fileCacheTestSuite) createRemoteDirectoryStructure() { suite.assert.NoError(err) } +// TestOpenFileDownloadFailure tests that when CopyToFile fails during OpenFile, +// the correct download error is returned (not the cleanup error) and the +// partially downloaded file is removed from the cache. +func (suite *fileCacheTestSuite) TestOpenFileDownloadFailure() { + // Create a mock controller and component + mockCtrl := gomock.NewController(suite.T()) + defer mockCtrl.Finish() + + mockComponent := internal.NewMockComponent(mockCtrl) + + // Create a new file cache with the mock component + // First, stop the default file cache + err := suite.fileCache.Stop() + suite.assert.NoError(err) + err = suite.loopback.Stop() + suite.assert.NoError(err) + + randStr := randomString(8) + cachePath := filepath.Join(home_dir, "file_cache"+randStr) + defaultConfig := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n timeout-sec: 0", cachePath) + + err = config.ReadConfigFromReader(strings.NewReader(defaultConfig)) + suite.assert.NoError(err) + + fileCache := NewFileCacheComponent() + fileCache.SetNextComponent(mockComponent) + err = fileCache.Configure(true) + suite.assert.NoError(err) + + // Expect Start and Stop calls + mockComponent.EXPECT().Start(gomock.Any()).Return(nil).Times(1) + mockComponent.EXPECT().Stop().Return(nil).Times(1) + + err = mockComponent.Start(context.Background()) + suite.assert.NoError(err) + + err = fileCache.Start(context.Background()) + suite.assert.NoError(err) + + fc := fileCache.(*FileCache) + + // Test file path + path := "test_download_failure.txt" + localPath := filepath.Join(cachePath, path) + + // Set up expectations for GetAttr to return a valid file that exists in storage + mockComponent.EXPECT(). + GetAttr(gomock.Any()). + Return(&internal.ObjAttr{ + Path: path, + Name: filepath.Base(path), + Size: 1024, + Mode: 0644, + Flags: internal.NewFileBitMap(), + }, nil). + Times(1) + + // Set up expectation for CopyToFile to fail with a download error + downloadErr := fmt.Errorf("simulated download failure") + mockComponent.EXPECT(). + CopyToFile(gomock.Any()). + Return(downloadErr). + Times(1) + + // Attempt to open the file - this should fail + handle, err := fc.OpenFile(internal.OpenFileOptions{Name: path, Mode: 0644}) + + // Assert that the error returned is the download error, not a cleanup error + suite.assert.Error(err) + suite.assert.Equal(downloadErr, err) + suite.assert.Nil(handle) + + // Verify that the partially downloaded file was cleaned up from the cache + _, statErr := os.Stat(localPath) + suite.assert.True(os.IsNotExist(statErr), "Partially downloaded file should be cleaned up from cache") + + // Clean up + err = fc.Stop() + suite.assert.NoError(err) + err = mockComponent.Stop() + suite.assert.NoError(err) + os.RemoveAll(cachePath) + + // Restart the default file cache for other tests + suite.loopback = newLoopbackFS() + suite.fileCache = newTestFileCache(suite.loopback) + err = suite.loopback.Start(context.Background()) + suite.assert.NoError(err) + err = suite.fileCache.Start(context.Background()) + suite.assert.NoError(err) +} + // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestFileCacheTestSuite(t *testing.T) { From 3a1bde1c68c909572341fbc054a19b18c21894fc Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Fri, 20 Feb 2026 16:00:07 +0530 Subject: [PATCH 10/95] Ensure logger is destroyed before returning from mount command. (#2125) --- CHANGELOG.md | 1 + cmd/mount.go | 6 ++++++ main.go | 12 ------------ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7add33075..3b583b316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ **Bug Fixes** - Fix panic while reading an archived blob using file-cache ([PR #2127](https://github.com/Azure/azure-storage-fuse/pull/2127)] +- Ensure logger is properly deinitialized on mount failures ([PR #2125](https://github.com/Azure/azure-storage-fuse/pull/2125)] ## 2.5.2 (2026-01-19) **Features** diff --git a/cmd/mount.go b/cmd/mount.go index ec6c61668..4c9b64dc5 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -440,6 +440,12 @@ var mountCmd = &cobra.Command{ return fmt.Errorf("failed to initialize logger [%s]", err.Error()) } + // It's best to destroy the logger before we return to the caller, as caller may abruptly exit on error and we + // might lose some logs in the channel which are not yet flushed to the file in case of not destroying the logger + defer func() { + _ = log.Destroy() + }() + if !disableVersionCheck { err := VersionCheck() if err != nil { diff --git a/main.go b/main.go index 8860f6ce3..380d1c271 100644 --- a/main.go +++ b/main.go @@ -35,23 +35,11 @@ package main import ( "github.com/Azure/azure-storage-fuse/v2/cmd" - "github.com/Azure/azure-storage-fuse/v2/common/log" ) //go:generate ./cmd/componentGenerator.sh $NAME // To use go:generate run command "NAME="component" go generate" func main() { - defer log.Destroy() // nolint:errcheck - // This recovers the panics only for the functions that run within this context. all the go-routine - // spawned by this function need to handle their panics separately if required. Also the FUSE callbacks - // wouldn't run in this context, so the panics originated from the callbacks can't get recovered here. - defer func() { - if panicErr := recover(); panicErr != nil { - log.Crit("PANIC: %v", panicErr) - panic(panicErr) - } - }() - _ = cmd.Execute() } From e1d861871322a9cbfad9a8ff0dd19fb53558444d Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Mar 2026 10:36:39 +0530 Subject: [PATCH 11/95] Fix CVE-2025-68121: Update Go toolchain to 1.25.7 (#2126) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: syeleti-msft <172502481+syeleti-msft@users.noreply.github.com> Co-authored-by: syeleti-msft --- .github/copilot-instructions.md | 6 +++--- go.mod | 2 +- go_installer.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 197f061a4..cc032f00f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -16,9 +16,9 @@ Always reference these instructions first and fallback to search or bash command sudo apt install -y libfuse3-dev fuse3 gcc ``` -- Install Go 1.24.4+ (already available in most environments): +- Install Go 1.25.7+ (already available in most environments): ```bash - go version # Should show 1.24.4 or higher + go version # Should show 1.25.7 or higher ``` - Build blobfuse2 binary: @@ -108,7 +108,7 @@ Always reference these instructions first and fallback to search or bash command - `./build.sh fuse2` - legacy fuse2 build - `./build.sh health` - health monitor binary - **Output**: `blobfuse2` binary (~30MB) and optionally `bfusemon` binary (~6MB) -- **Go Version**: Requires Go 1.24.4+ (specified in go.mod) +- **Go Version**: Requires Go 1.25.7+ (specified in go.mod) - **Tags**: Use `fuse3` tag for testing/building (default), `fuse2` for legacy systems ## Testing Infrastructure diff --git a/go.mod b/go.mod index 5bf9ae6d2..b44e2338a 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/Azure/azure-storage-fuse/v2 go 1.25 -toolchain go1.25.6 +toolchain go1.25.7 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 diff --git a/go_installer.sh b/go_installer.sh index 0540acc6a..3dae86720 100755 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,6 +1,6 @@ #!/bin/bash work_dir=$(echo $1 | sed 's:/*$::') -version="1.25.6" +version="1.25.7" arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` if [ $arch != "arm64" ] From 67bfedc1842282208b41d3875938ec8757bc0aa7 Mon Sep 17 00:00:00 2001 From: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> Date: Tue, 3 Mar 2026 14:43:27 +0530 Subject: [PATCH 12/95] Add script to consolidate data and upload to storage account (#2143) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> --- .github/workflows/mirror-to-blob.yml | 100 ++++++++++ .gitignore | 6 +- scripts/export_github_history.py | 262 +++++++++++++++++++++++++++ 3 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/mirror-to-blob.yml create mode 100644 scripts/export_github_history.py diff --git a/.github/workflows/mirror-to-blob.yml b/.github/workflows/mirror-to-blob.yml new file mode 100644 index 000000000..3d320f999 --- /dev/null +++ b/.github/workflows/mirror-to-blob.yml @@ -0,0 +1,100 @@ +name: Step-2 Mirror GitHub Content to Blob Storage + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * 0" # every Sunday at 00:00 UTC + +permissions: + id-token: write + contents: read + issues: read + pull-requests: read + +jobs: + mirror: + runs-on: ubuntu-latest + timeout-minutes: 180 # ✅ 3 hours + + env: + AZURE_STORAGE_ACCOUNT: blobfusemcpserver + + steps: + - uses: actions/checkout@v4 + + # ✅ Azure authentication (OIDC) + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_MCP_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_MCP_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_MCP_SUBSCRIPTION_ID }} + + # ✅ Upload repo code + - name: Upload repo code + run: | + az storage copy \ + --auth-mode login \ + --account-name $AZURE_STORAGE_ACCOUNT \ + --source . \ + --destination "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/repo-code/azure-storage-fuse" \ + --recursive \ + --exclude-path ".git" + + # ✅ Upload wiki + - name: Clone wiki + run: | + git clone https://github.com/Azure/azure-storage-fuse.wiki.git wiki + + - name: Upload wiki docs + run: | + az storage blob upload-batch \ + --auth-mode login \ + --account-name $AZURE_STORAGE_ACCOUNT \ + --container-name repo-docs \ + --source wiki \ + --destination wiki + + # ✅ Export issues / PRs + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install requests + + # ✅ Restore incremental state and existing JSONL exports so the exporter + # can append new/updated records to the full historical files rather + # than to an empty workspace, preserving the complete history in blob. + - name: Restore incremental state + run: | + mkdir -p out_github_export + az storage blob download-batch \ + --auth-mode login \ + --account-name $AZURE_STORAGE_ACCOUNT \ + --source repo-gh \ + --destination out_github_export \ + --no-progress \ + || echo "No previous export in blob - first run will perform a full export" + + - name: Export GitHub history + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 scripts/export_github_history.py + + # ✅ Upload GitHub history (including updated state.json for next incremental run). + # --overwrite ensures all exported JSONL files and state.json are replaced with the + # latest data from this run. + - name: Upload GitHub history + run: | + az storage blob upload-batch \ + --auth-mode login \ + --account-name $AZURE_STORAGE_ACCOUNT \ + --container-name repo-gh \ + --source out_github_export \ + --destination . \ + --overwrite diff --git a/.gitignore b/.gitignore index 4f2bea89e..5ee43a215 100755 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,8 @@ _tmp_*.trace # Emacs backup and auto-save files *~ \#*\# -.\#* \ No newline at end of file +.\#* +# Python bytecode +__pycache__/ +*.pyc +*.pyo diff --git a/scripts/export_github_history.py b/scripts/export_github_history.py new file mode 100644 index 000000000..c0d280e57 --- /dev/null +++ b/scripts/export_github_history.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +""" +Export GitHub repo issues + PRs + comments into JSONL files suitable for Azure AI Search blob indexer +using parsingMode: jsonLines (each line = one JSON document). + +Repo targeted: Azure/azure-storage-fuse (can be overridden by env vars OWNER/REPO). + +Outputs (in OUT_DIR, default: out_github_export): + - threads_issues_prs.jsonl (one doc per issue or PR thread) + - issue_pr_comments.jsonl (one doc per issue/PR timeline comment) + - pr_reviews.jsonl (one doc per PR review) + - pr_review_comments.jsonl (one doc per PR diff review comment) + +Incremental: + Stores latest updated_at in out_github_export/state.json and uses it as "since" next run. +""" + +import os +import sys +import json +import time +import hashlib +from urllib.parse import urlencode + +import requests + +OWNER = os.getenv("OWNER", "Azure") +REPO = os.getenv("REPO", "azure-storage-fuse") +BASE = "https://api.github.com" + +# Optional: set GITHUB_TOKEN to increase rate limits +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "").strip() + +OUT_DIR = os.getenv("OUT_DIR", "out_github_export") +STATE_FILE = os.path.join(OUT_DIR, "state.json") + + +def _headers(): + h = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "blobfuse-support-exporter", + } + if GITHUB_TOKEN: + h["Authorization"] = f"Bearer {GITHUB_TOKEN}" + return h + + +def _ensure_out_dir(): + os.makedirs(OUT_DIR, exist_ok=True) + + +def _load_state(): + if os.path.exists(STATE_FILE): + with open(STATE_FILE, "r", encoding="utf-8") as f: + return json.load(f) + return {"since": None} + + +def _save_state(state): + with open(STATE_FILE, "w", encoding="utf-8") as f: + json.dump(state, f, indent=2) + + +def _request_json(url, params=None, max_retries=3): + if params: + url = f"{url}?{urlencode(params)}" + + retries = 0 + while True: + r = requests.get(url, headers=_headers(), timeout=60) + + # Basic rate limit handling + if r.status_code == 403 and "rate limit" in r.text.lower(): + reset = r.headers.get("X-RateLimit-Reset") + if reset: + sleep_for = max(1, int(reset) - int(time.time()) + 5) + print(f"[rate-limit] sleeping {sleep_for}s", file=sys.stderr) + time.sleep(sleep_for) + continue + + # Retry on transient 5xx errors + if r.status_code >= 500 and retries < max_retries: + retries += 1 + sleep_for = 2 ** retries # exponential backoff: 2, 4, 8 seconds + print(f"[retry {retries}/{max_retries}] {r.status_code} - sleeping {sleep_for}s", file=sys.stderr) + time.sleep(sleep_for) + continue + + r.raise_for_status() + return r.json(), r.headers + + +def _paginate(url, params=None): + params = params or {} + page = 1 + while True: + p = dict(params) + p["per_page"] = 100 + p["page"] = page + data, _hdrs = _request_json(url, p) + if not data: + break + for item in data: + yield item + page += 1 + + +def _stable_id(*parts): + raw = "|".join(str(p) for p in parts) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def export(): + _ensure_out_dir() + state = _load_state() + since = state.get("since") # ISO8601 timestamp or None + + # GitHub "issues" list includes PRs; PRs have "pull_request" key. + issues_url = f"{BASE}/repos/{OWNER}/{REPO}/issues" + params = {"state": "all", "sort": "updated", "direction": "desc"} + if since: + params["since"] = since + + print("Fetching issues from:", issues_url, params) + + # Open output files once for streaming writes. + # Use append mode on incremental runs so previously exported records are + # preserved; use write mode on a full (first) run or when prior JSONL + # files are not present (e.g., fresh CI workspace) to avoid creating + # partial histories. Azure AI Search uses the stable document "id" for + # upserts, so duplicate records in append mode are safely overwritten by + # the indexer. + threads_path = os.path.join(OUT_DIR, "threads_issues_prs.jsonl") + comments_path = os.path.join(OUT_DIR, "issue_pr_comments.jsonl") + reviews_path = os.path.join(OUT_DIR, "pr_reviews.jsonl") + review_comments_path = os.path.join(OUT_DIR, "pr_review_comments.jsonl") + + if since and all(os.path.exists(p) for p in (threads_path, comments_path, reviews_path, review_comments_path)): + file_mode = "a" + else: + file_mode = "w" + newest = None + thread_count = 0 + + with open(threads_path, file_mode, encoding="utf-8") as f_threads, \ + open(comments_path, file_mode, encoding="utf-8") as f_comments, \ + open(reviews_path, file_mode, encoding="utf-8") as f_reviews, \ + open(review_comments_path, file_mode, encoding="utf-8") as f_review_comments: + + for it in _paginate(issues_url, params): + thread_count += 1 + number = it["number"] + is_pr = "pull_request" in it + print("Fetching id: ", number, " PR: ", is_pr) + + labels = [] + if isinstance(it.get("labels"), list): + labels = [l.get("name") for l in it.get("labels", []) if isinstance(l, dict) and l.get("name")] + + # 1) thread document (issue or PR) — written immediately + f_threads.write(json.dumps({ + "id": _stable_id("thread", f"{OWNER}/{REPO}#{number}"), + "content_type": "github_pr" if is_pr else "github_issue", + "repo": f"{OWNER}/{REPO}", + "github_number": number, + "title": it.get("title", ""), + "content": it.get("body") or "", + "state": it.get("state"), + "labels": labels, + "author": (it.get("user") or {}).get("login"), + "created_at": it.get("created_at"), + "updated_at": it.get("updated_at"), + "closed_at": it.get("closed_at"), + "source_url": it.get("html_url"), + }, ensure_ascii=False) + "\n") + + # Track newest timestamp for incremental state update + ts = it.get("updated_at") + if ts and (newest is None or ts > newest): + newest = ts + + # 2) issue comments (for issues AND PR timeline comments) + ic_url = f"{BASE}/repos/{OWNER}/{REPO}/issues/{number}/comments" + for c in _paginate(ic_url, {"since": since} if since else None): + f_comments.write(json.dumps({ + "id": _stable_id("issue_comment", number, c["id"]), + "content_type": "github_pr_issue_comment" if is_pr else "github_issue_comment", + "repo": f"{OWNER}/{REPO}", + "github_number": number, + "comment_id": c["id"], + "author": (c.get("user") or {}).get("login"), + "created_at": c.get("created_at"), + "updated_at": c.get("updated_at"), + "source_url": c.get("html_url"), + "content": c.get("body") or "", + }, ensure_ascii=False) + "\n") + + if is_pr: + # 3) PR reviews (top-level review bodies) + # The GitHub reviews endpoint does not support a 'since' filter; + # skip reviews whose submitted_at is before the incremental cursor. + reviews_url = f"{BASE}/repos/{OWNER}/{REPO}/pulls/{number}/reviews" + for rv in _paginate(reviews_url): + submitted_at = rv.get("submitted_at") + if since and submitted_at and submitted_at < since: + continue + f_reviews.write(json.dumps({ + "id": _stable_id("pr_review", number, rv["id"]), + "content_type": "github_pr_review", + "repo": f"{OWNER}/{REPO}", + "github_number": number, + "review_id": rv["id"], + "state": rv.get("state"), + "author": (rv.get("user") or {}).get("login"), + "submitted_at": submitted_at, + "source_url": rv.get("html_url"), + "content": rv.get("body") or "", + }, ensure_ascii=False) + "\n") + + # 4) PR review comments (diff comments) + # The GitHub pull review comments endpoint supports 'since', + # so pass the incremental cursor to avoid re-fetching old records. + prc_url = f"{BASE}/repos/{OWNER}/{REPO}/pulls/{number}/comments" + prc_params = {"since": since} if since else {} + for rc in _paginate(prc_url, prc_params): + f_review_comments.write(json.dumps({ + "id": _stable_id("pr_review_comment", number, rc["id"]), + "content_type": "github_pr_review_comment", + "repo": f"{OWNER}/{REPO}", + "github_number": number, + "comment_id": rc["id"], + "pull_request_review_id": rc.get("pull_request_review_id"), + "path": rc.get("path"), + "line": rc.get("line"), + "side": rc.get("side"), + "author": (rc.get("user") or {}).get("login"), + "created_at": rc.get("created_at"), + "updated_at": rc.get("updated_at"), + "source_url": rc.get("html_url"), + "content": rc.get("body") or "", + }, ensure_ascii=False) + "\n") + + # Flush after each thread so progress is preserved on interruption + f_threads.flush() + f_comments.flush() + f_reviews.flush() + f_review_comments.flush() + + print("Threads fetched:", thread_count) + + # Update incremental marker: latest updated_at seen + if newest: + state["since"] = newest + _save_state(state) + + print(f"[ok] wrote exports to: {OUT_DIR}") + print(f"[ok] next incremental since: {state.get('since')}") + + +if __name__ == "__main__": + export() \ No newline at end of file From 4da2b58ff77a12eb5393ebc8e8c13b4d47421646 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Mar 2026 10:54:35 +0530 Subject: [PATCH 13/95] Bump actions/checkout from 4 to 6 (#2149) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/mirror-to-blob.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mirror-to-blob.yml b/.github/workflows/mirror-to-blob.yml index 3d320f999..c8bdefc1a 100644 --- a/.github/workflows/mirror-to-blob.yml +++ b/.github/workflows/mirror-to-blob.yml @@ -20,7 +20,7 @@ jobs: AZURE_STORAGE_ACCOUNT: blobfusemcpserver steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 # ✅ Azure authentication (OIDC) - name: Azure login From 9149a1a6499fe690532bea7213c6db414adaa81d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Mar 2026 11:23:01 +0530 Subject: [PATCH 14/95] Bump actions/setup-python from 5 to 6 (#2150) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/mirror-to-blob.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mirror-to-blob.yml b/.github/workflows/mirror-to-blob.yml index c8bdefc1a..61fcf9a62 100644 --- a/.github/workflows/mirror-to-blob.yml +++ b/.github/workflows/mirror-to-blob.yml @@ -57,7 +57,7 @@ jobs: # ✅ Export issues / PRs - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.x" From 8ad0f9491e50e0a1189701578d2a9e58b6d5cf3c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:49:24 +0530 Subject: [PATCH 15/95] Bump golang.org/x/time from 0.14.0 to 0.15.0 (#2152) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Srinivas Yeleti --- component/libfuse/libfuse2_handler.go | 2 +- component/libfuse/libfuse_handler.go | 2 +- component/libfuse/libfuse_handler_test.go | 2 +- go.mod | 4 ++-- go.sum | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/component/libfuse/libfuse2_handler.go b/component/libfuse/libfuse2_handler.go index fd1165ba5..541162fd1 100644 --- a/component/libfuse/libfuse2_handler.go +++ b/component/libfuse/libfuse2_handler.go @@ -124,7 +124,7 @@ func (lf *Libfuse) initFuse() error { operations := C.fuse_operations_t{} if lf.extensionPath != "" { - log.Trace("Libfuse::InitFuse : Going for extension mouting [%s]", lf.extensionPath) + log.Trace("Libfuse::InitFuse : Going for extension mounting [%s]", lf.extensionPath) // User has given an extension so we need to register it to fuse // and then register ourself to it diff --git a/component/libfuse/libfuse_handler.go b/component/libfuse/libfuse_handler.go index bcff3b65e..ff1e6ce1a 100644 --- a/component/libfuse/libfuse_handler.go +++ b/component/libfuse/libfuse_handler.go @@ -123,7 +123,7 @@ func (lf *Libfuse) initFuse() error { operations := C.fuse_operations_t{} if lf.extensionPath != "" { - log.Trace("Libfuse::InitFuse : Going for extension mouting [%s]", lf.extensionPath) + log.Trace("Libfuse::InitFuse : Going for extension mounting [%s]", lf.extensionPath) // User has given an extension so we need to register it to fuse // and then register ourself to it diff --git a/component/libfuse/libfuse_handler_test.go b/component/libfuse/libfuse_handler_test.go index 51ef953db..80f0d8944 100644 --- a/component/libfuse/libfuse_handler_test.go +++ b/component/libfuse/libfuse_handler_test.go @@ -152,7 +152,7 @@ func (suite *libfuseTestSuite) TestConfigFuseTraceEnable() { suite.assert.Equal("libfuse", suite.libfuse.Name()) suite.assert.Empty(suite.libfuse.mountPath) - // Fuse trace should work as we are mouting using foreground option. + // Fuse trace should work as we are mounting using foreground option. suite.assert.True(suite.libfuse.traceEnable) common.ForegroundMount = false } diff --git a/go.mod b/go.mod index b44e2338a..4ce19bb9b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Azure/azure-storage-fuse/v2 -go 1.25 +go 1.25.0 toolchain go1.25.7 @@ -24,7 +24,7 @@ require ( github.com/vibhansa-msft/blobfilter v0.0.0-20250115104552-d9d40722be3e github.com/vibhansa-msft/tlru v0.0.0-20240410102558-9e708419e21f go.uber.org/atomic v1.11.0 - golang.org/x/time v0.14.0 + golang.org/x/time v0.15.0 gopkg.in/ini.v1 v1.67.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index a7b06bbde..850e3de9f 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= From 0ce07935ffed232f5275f71e25e7000b8767e8c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:56:04 +0530 Subject: [PATCH 16/95] Bump golang.org/x/sys from 0.41.0 to 0.42.0 (#2153) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4ce19bb9b..b5e5f26e2 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.50.0 // indirect - golang.org/x/sys v0.41.0 + golang.org/x/sys v0.42.0 golang.org/x/text v0.34.0 // indirect ) diff --git a/go.sum b/go.sum index 850e3de9f..cb22c5f50 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 282ccfaf4f8a738d416ae1efe258ae32816ff466 Mon Sep 17 00:00:00 2001 From: NISHANT RANJAN <115216024+p15nishatr@users.noreply.github.com> Date: Mon, 9 Mar 2026 12:10:02 +0530 Subject: [PATCH 17/95] Update README.md (#1934) Co-authored-by: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> --- README.md | 372 +++--------------------------------------------------- 1 file changed, 18 insertions(+), 354 deletions(-) diff --git a/README.md b/README.md index 660e4003d..bfac2460a 100755 --- a/README.md +++ b/README.md @@ -1,370 +1,34 @@ -# Blobfuse2 - A Microsoft supported Azure Storage FUSE driver +# Blobfuse - A Microsoft supported Azure Storage FUSE driver ## About -Blobfuse2 is an open source project developed to provide a virtual filesystem backed by the Azure Storage. It uses the libfuse open source library (fuse3) to communicate with the Linux FUSE kernel module, and implements the filesystem operations using the Azure Storage REST APIs. -This is the next generation [blobfuse](https://github.com/Azure/azure-storage-fuse). -## About Data Consistency and Concurrency -Blobfuse2 is stable and ***supported by Microsoft*** when used within its [documented limits](#un-supported-file-system-operations). Blobfuse2 supports high-performance reads and writes with strong consistency; however, it is recommended that multiple clients do not modify the same blob/file simultaneously to ensure data integrity. Blobfuse2 does not guarantee continuous synchronization of data written to the same blob/file using multiple clients or across multiple mounts of Blobfuse2 concurrently. If you modify an existing blob/file with another client while also reading that object, Blobfuse2 will not return the most up-to-date data. To ensure your reads see the newest blob/file data, disable all forms of caching at kernel (using `direct-io`) as well as at Blobfuse2 level, and then re-open the blob/file. +Blobfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. It uses the libfuse open source library (fuse3) to communicate with the Linux FUSE kernel module, and implements the filesystem operations using the Azure Storage REST APIs. BlobFuse lets Linux workloads use Azure Blob Storage like a local file system for several scenarios, including AI/ML training and checkpointing, HPC simulations, Kubernetes stateful workloads, big data analytics/preprocessing, and large-scale backup and archiving. BlobFuse supports standard file operations with caching, offers health monitoring and blob filtering, and can preload containers or folders into the local cache for faster access. -Please submit an issue [here](https://github.com/azure/azure-storage-fuse/issues) for any issues/feature requests/questions. +BlobFuse supports two operating modes: a **caching** mode that works by caching data locally on VM nodes, ideal for repeatedly accessed data that fits on VM local storage, and a **streaming** mode that reads large files in chunks directly from storage for big AI/ML, genomics, or HPC workloads. This [article](https://learn.microsoft.com/azure/storage/blobs/blobfuse2-streaming-versus-caching) helps you decide which mode is best suited for your workloads. +> [!NOTE] +> BlobFuse v2 is the latest version of BlobFuse and has many significant improvements over BlobFuse v1. BlobFuse v1 support ends in September 2026. Migrate to BlobFuse v2 by using the provided [instructions](https://github.com/Azure/azure-storage-fuse/blob/main/MIGRATION.md). -[This](#config-guide) section will help you choose the correct config for Blobfuse2. +Detailed documentation of BlobFuse is [here](https://learn.microsoft.com/azure/storage/blobs/blobfuse2-what-is). +Please submit any issues/feature requests/questions [here](https://github.com/azure/azure-storage-fuse/issues). -## NOTICE -- Due to known data consistency issues when using Blobfuse2 in `block-cache` mode, it is strongly recommended that all Blobfuse2 installations be upgraded to version 2.3.2. For more information, see [this](https://github.com/Azure/azure-storage-fuse/wiki/Blobfuse2-Known-issues). -- Login via Managed Identify is supported with Object-ID for all versions of Blobfuse except 2.3.0 and 2.3.2.To use Object-ID for these two versions, use Azure CLI or utilize Application/Client-ID or Resource ID based authentication. -- `streaming` mode is being deprecated. This is the older option and is replaced by streaming with `block-cache` mode which is the more performant streaming option. -- Block cache will no longer dynamically consume more memory if required by application but will strictly adhere to the memory limit which is 80% of free memory by default or whatever is configured by the user. +## Install BlobFuse +You can install BlobFuse from Microsoft repositories for Linux by using simple commands to install the BlobFuse package. If no package is available for your distribution and version, you can build the binary from source code. Refer to the [instructions](https://learn.microsoft.com/azure/storage/blobs/blobfuse2-mount-container) for details. -## Limitations in Block Cache -- Concurrent write operations on the same file using multiple handles is not checked for data consistency and may lead to incorrect data being written. -- A read operation on a file that is being written to simultaneously by another process or handle will not return the most up-to-date data. -- When copying files with trailing null bytes using `cp` utility to a Blobfuse2 mounted path, use `--sparse=never` parameter to avoid data being trimmed. For example, `cp --sparse=never src dest`. -- In write operations, data written is persisted (or committed) to the Azure Storage container only when close, sync or flush operations are called by user application. -- Files cannot be modified if they were originally created with block-size different than the one configured. +## Mount BlobFuse +You can mount a container by using the `mount` command. You can either include your desired configuration settings as command line parameters or provide a configuration file that contains your settings. Refer the [page](https://learn.microsoft.com/azure/storage/blobs/blobfuse2-install) for details. -## Recommendations in Block Cache -- User applications must check the returned code (success/failure) for filesystem calls like read, write, close, flush, etc. If error is returned, the application must abort their respective operation. -- User applications must ensure that there is only one writer at a time for a given file. -- When dealing with very large files (in TiB), the block-size must be configured accordingly. Azure Storage supports only [50,000 blocks](https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list?tabs=microsoft-entra-id#remarks) per blob. - -## Blobfuse2 Benchmarks -[This](https://azure.github.io/azure-storage-fuse/) page lists various benchmarking results for HNS and FNS Storage account. +## BlobFuse Logging +By default, BlobFuse logs warnings to the system log. However, you can route logs to a local directory, change which types of information appear in logs, or disable logs entirely by changing the default configuration. Refer the [page](https://learn.microsoft.com/azure/storage/blobs/blobfuse2-enable-logs) for details. -## Supported Platforms -Visit [this](https://github.com/Azure/azure-storage-fuse/wiki/Blobfuse2-Supported-Platforms) page to see list of supported linux distros. - -## Features -- Mount an Azure storage blob container or datalake file system on Linux. -- Basic file system operations such as mkdir, opendir, readdir, rmdir, open, - read, create, write, close, unlink, truncate, stat, rename -- Local caching to improve subsequent access times -- Block-Cache to support reading AND writing large files -- Parallel downloads and uploads to improve access time for large files -- Multiple mounts to the same container for read-only workloads - -## _New BlobFuse2 Health Monitor_ -One of the biggest BlobFuse2 features is our brand new health monitor. It allows customers gain more insight into how their BlobFuse2 instance is behaving with the rest of their machine. Visit [here](https://github.com/Azure/azure-storage-fuse/blob/main/tools/health-monitor/README.md) to set it up. - -## Distinctive features compared to blobfuse (v1.x) -- Blobfuse2 is fuse3 compatible (other than Ubuntu-18 and Debian-9, where it still runs with fuse2) -- Support for higher service version offering latest and greatest of azure storage features (supported by azure go-sdk) -- Set blob tier while uploading the data to storage -- Attribute cache invalidation based on timeout -- For flat namespace accounts, user can configure default permissions for files and folders -- Improved cache eviction algorithm for file cache to control disk footprint of blobfuse2 -- Improved cache eviction algorithm for streamed buffers to control memory footprint of blobfuse2 -- Utility to convert blobfuse CLI and config parameters to a blobfuse2 compatible config for easy migration -- CLI to mount Blobfuse2 with legacy Blobfuse config and CLI parameters (Refer to Migration guide for this) -- Version check and upgrade prompting -- Option to mount a sub-directory from a container -- CLI to mount all containers (with a allowlist and denylist) in a given storage account -- CLI to list all blobfuse2 mount points -- CLI to unmount one, multiple or all blobfuse2 mountpoints -- Option to dump logs to syslog or a file on disk -- Support for config file encryption and mounting with an encrypted config file via a passphrase (CLI or environment variable) to decrypt the config file -- CLI to check or update a parameter in the encrypted config -- Set MD5 sum of a blob while uploading -- Validate MD5 sum on download and fail file open on mismatch -- Large file writing through write Block-Cache -- Blob filter to view only files matching given criteria for read-only mount - - ## Blobfuse2 performance compared to blobfuse(v1.x.x) -- 'git clone' operation is 25% faster (tested with vscode repo cloning) -- ResNet50 image classification job is 7-8% faster (tested with 1.3 million images) -- Regular file uploads are 10% faster -- Verified listing of 1-Billion files in a directory (which v1.x does not support) - - -## Download Blobfuse2 -You can install Blobfuse2 by cloning this repository. In the workspace root execute below commands to build the binary. - -- sudo apt install fuse3 libfuse3-dev gcc -- go build -o blobfuse2 - - - - -## Supported Operations -The general format of the Blobfuse2 commands is `blobfuse2 [command] [arguments] --[flag-name]=[flag-value]` -* `help` - Help about any command -* `mount` - Mounts an Azure container as a filesystem. The supported containers include - - Azure Blob Container - - Azure Datalake Gen2 Container -* `mount all` - Mounts all the containers in an Azure account as a filesystem. The supported storage services include - - [Blob Storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - - [Datalake Storage Gen2](https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction) -* `mount list` - Lists all Blobfuse2 filesystems. -* `secure decrypt` - Decrypts a config file. -* `secure encrypt` - Encrypts a config file. -* `secure get` - Gets value of a config parameter from an encrypted config file. -* `secure set` - Updates value of a config parameter. -* `unmount` - Unmounts the Blobfuse2 filesystem. -* `unmount all` - Unmounts all Blobfuse2 filesystems. -* `unmount all --lazy` - Unmounts all Blobfuse2 filesystems in lazy unmount mode. -* `gen-config` - Auto generate recommended blobfuse2 config file. +## Monitor BlobFuse +Health monitor is a tool that you use to monitor mount activities and resource usage. This [article](https://learn.microsoft.com/azure/storage/blobs/blobfuse2-health-monitor) describes what data you can obtain, and how to enable health monitor and view output reports. ## Find help from your command prompt To see a list of commands, type `blobfuse2 -h` and then press the ENTER key. To learn about a specific command, just include the name of the command (For example: `blobfuse2 mount -h`). -## Usage -- Mount with blobfuse2 - * blobfuse2 mount \ --config-file=\ -- Mount blobfuse2 using legacy blobfuse config and cli parameters - * blobfuse2 mountv1 \ -- Mount all containers in your storage account - * blobfuse2 mount all \ --config-file=\ -- List all mount instances of blobfuse2 - * blobfuse2 mount list -- Unmount blobfuse2 - * sudo fusermount3 -u \ -- Unmount blobfuse2 in lazy mode - * sudo fusermount3 -u \ --lazy -- Unmount all blobfuse2 instances - * blobfuse2 unmount all -- Auto generate config file - * blobfuse2 gen-config --tmp-path=\ --o \ - * blobfuse2 gen-config --block-cache --o \ - - -## CLI parameters -- Note: Blobfuse2 accepts all CLI parameters that Blobfuse does, but may ignore parameters that are no longer applicable. -- General options - * `--config-file=`: The path to the config file. - * `--log-level=`: The level of logs to capture. - * `--log-file-path=`: The path for the log file. - * `--foreground=true`: Mounts the system in foreground mode. - * `--read-only=true`: Mount container in read-only mode. - * `--default-working-dir`: The default working directory to store log files and other blobfuse2 related information. - * `--disable-version-check=true`: Disable the blobfuse2 version check. - * `--secure-config=true` : Config file is encrypted suing 'blobfuse2 secure` command. - * `--passphrase=` : Passphrase used to encrypt/decrypt config file. - * `--wait-for-mount=` : Let parent process wait for given timeout before exit to ensure child has started. - * `--block-cache` : To enable block-cache instead of file-cache. This works only when mounted without any config file. - * `--filter=`: Enable blob filters for read-only mount to restrict the view on what all blobs user can see or read. - * `--preload`: Enable preload for read-only mount to start downloading all blobs from container when mount succeeds. - -- Attribute cache options - * `--attr-cache-timeout=`: The timeout for the attribute cache entries. - * `--no-symlinks=false`: By default symlinks will be supported and the performance overhead, that earlier existed, has been resolved. -- Storage options - * `--container-name=`: The container to mount. - * `--cancel-list-on-mount-seconds=`: Time for which list calls will be blocked after mount. ( prevent billing charges on mounting) - * `--virtual-directory=true` : Support virtual directories without existence of a special marker blob for block blob account. - * `--subdirectory=` : Subdirectory to mount instead of entire container. - * `--disable-compression:false` : Disable content encoding negotiation with server. If blobs have 'content-encoding' set to 'gzip' then turn on this flag. - * `--use-adls=false` : Specify configured storage account is HNS enabled or not. This must be turned on when HNS enabled account is mounted. - * `--cpk-enabled=true`: Allows mounting containers with cpk. Use config file or env variables to set cpk encryption key and cpk encryption key sha. -- File cache options - * `--file-cache-timeout=`: Timeout for which file is cached on local system. - * `--tmp-path=`: The path to the file cache. - * `--cache-size-mb=`: Amount of disk cache that can be used by blobfuse. Default - 80% of free disk space. - * `--high-disk-threshold=`: If local cache usage exceeds this, start early eviction of files from cache. - * `--low-disk-threshold=`: If local cache usage comes below this threshold then stop early eviction. - * `--sync-to-flush=false` : Sync call will force upload a file to storage container if this is set to true, otherwise it just evicts file from local cache. -- Block-Cache options - * `--block-cache-block-size=`: Size of a block to be downloaded as a unit. - * `--block-cache-pool-size=`: Size of pool to be used for caching. This limits total memory used by block-cache. Default - 80% of free memory available. - * `--block-cache-path=`: Path where downloaded blocks will be persisted. Not providing this parameter will disable the disk caching. - * `--block-cache-disk-size=`: Disk space to be used for caching. Default - 80% of free disk space. - * `--block-cache-disk-timeout=`: Timeout for which disk cache is valid. - * `--block-cache-prefetch=`: Number of blocks to prefetch at max when sequential reads are in progress. Default - 2 times number of CPU cores. - * `--block-cache-parallelism=`: Number of parallel threads doing upload/download operation. Default - 3 times number of CPU cores. - * `--block-cache-prefetch-on-open=true`: Start prefetching on open system call instead of waiting for first read. Enhances perf if file is read sequentially from offset 0. - * `--block-cache-strong-consistency=true`: Enable strong data consistency checks in block-cache. This will increase load on your CPU and may introduce some latency. - This will need support of `xattr` on your system. Kindly install the feature manually before using this cli parameter. -- Fuse options - * `--attr-timeout=`: Time the kernel can cache inode attributes. - * `--entry-timeout=`: Time the kernel can cache directory listing. - * `--negative-timeout=`: Time the kernel can cache non-existance of file or directory. - * `--allow-other`: Allow other users to have access this mount point. - * `--disable-writeback-cache=true`: Disallow libfuse to buffer write requests if you must strictly open files in O_WRONLY or O_APPEND mode. - * `--ignore-open-flags=true`: Ignore the append and write only flag since O_APPEND and O_WRONLY is not supported with writeback caching. - - -## Environment variables -- General options - * `AZURE_STORAGE_ACCOUNT`: Specifies the storage account to be connected. - * `AZURE_STORAGE_ACCOUNT_TYPE`: Specifies the account type 'block' or 'adls' - * `AZURE_STORAGE_ACCOUNT_CONTAINER`: Specifies the name of the container to be mounted - * `AZURE_STORAGE_BLOB_ENDPOINT`: Specifies the blob endpoint to use. Defaults to *.blob.core.windows.net, but is useful for targeting storage emulators. - * `AZURE_STORAGE_AUTH_TYPE`: Overrides the currently specified auth type. Case insensitive. Options: Key, SAS, MSI, SPN -- Account key auth: - * `AZURE_STORAGE_ACCESS_KEY`: Specifies the storage account key to use for authentication. -- SAS token auth: - * `AZURE_STORAGE_SAS_TOKEN`: Specifies the SAS token to use for authentication. -- Managed Identity auth: - * `AZURE_STORAGE_IDENTITY_CLIENT_ID`: Only one of these three parameters are needed if multiple identities are present on the system. - * `AZURE_STORAGE_IDENTITY_OBJECT_ID`: Only one of these three parameters are needed if multiple identities are present on the system. - * `AZURE_STORAGE_IDENTITY_RESOURCE_ID`: Only one of these three parameters are needed if multiple identities are present on the system. - * `MSI_ENDPOINT`: Specifies a custom managed identity endpoint, as IMDS may not be available under some scenarios. Uses the `MSI_SECRET` parameter as the `Secret` header. - * `MSI_SECRET`: Specifies a custom secret for an alternate managed identity endpoint. -- Service Principal Name auth: - * `AZURE_STORAGE_SPN_CLIENT_ID`: Specifies the client ID for your application registration - * `AZURE_STORAGE_SPN_TENANT_ID`: Specifies the tenant ID for your application registration - * `AZURE_STORAGE_AAD_ENDPOINT`: Specifies a custom AAD endpoint to authenticate against - * `AZURE_STORAGE_SPN_CLIENT_SECRET`: Specifies the client secret for your application registration. - * `AZURE_STORAGE_AUTH_RESOURCE` : Scope to be used while requesting for token. -- Workload Identity auth: - * `AZURE_STORAGE_SPN_CLIENT_ID`: Specifies the clientid of the MI assigned to the storage account | clientid of the MI assigned as subject field on a Federated Identity Credential (FIC) on the App Registration - * `AZURE_STORAGE_SPN_TENANT_ID`: Specifies the tenant ID for your storage account - * `AZURE_STORAGE_IDENTITY_CLIENT_ID`: Specifies the application (client) ID of the App Registration or SPN - * `AZURE_STORAGE_AUTH_RESOURCE` : Scope to be used while requesting for token / MI Audience. - - Public Cloud: api://AzureADTokenExchange (Default) - - US Gov Cloud: api://AzureADTokenExchangeUSGov - - China Cloud operated by 21Vianet: api://AzureADTokenExchangeChina - -- Proxy Server: - * `http_proxy`: The proxy server address. Example: `10.1.22.4:8080`. - * `https_proxy`: The proxy server address when https is turned off forcing http. Example: `10.1.22.4:8080`. -- CPK options: - * `AZURE_STORAGE_CPK_ENCRYPTION_KEY`: Customer provided base64-encoded AES-256 encryption key value. - * `AZURE_STORAGE_CPK_ENCRYPTION_KEY_SHA256`: Base64-encoded SHA256 of the cpk encryption key. -- Custom component options: - * `BLOBFUSE_PLUGIN_PATH`: Specifies plugin file path as a colon-separated list of `.so` files. Example BLOBFUSE_PLUGIN_PATH="/path/to/plugin1.so:/path/to/plugin2.so". - - -## Config Guide -Below diagrams guide you to choose right configuration for your workloads. - -- Choose right Auth mode -

-![alt text](./guide/AuthModeHelper.png?raw=true "Auth Mode Selection Guide") -

-- Choose right caching for Read-Only workloads -

-![alt text](./guide/CacheModeForReadOnlyWorkloads.png?raw=true "Cache Mode Selection Guide For Read-Only Workloads") -

-- Choose right caching for Read-Write workloads -

-![alt text](./guide/CacheModeForReadWriteWorkloads.png?raw=true "Cache Mode Selection Guide For Read-Only Workloads") -

-- Choose right block-cache configuration -

-![alt text](./guide/BlockCacheConfig.png?raw=true "Block-Cache Configuration") -

-- Choose right file-cache configuration -

-![alt text](./guide/FileCacheConfig.png?raw=true "Block-Cache Configuration") -

-- [Sample File Cache Config](./sampleFileCacheConfig.yaml) -- [Sample Block-Cache Config](./sampleBlockCacheConfig.yaml) -- [All Config options](./setup/baseConfig.yaml) - -## Preload - -In file caching mode, Blobfuse waits for open file system call. On receiving the open call it downloads entire file to a local cache before using them. This can make the initial load slower, especially for AI/ML tasks, where application is processing many files.The Preload feature helps by downloading entire containers or sub-directories to the local cache when you mount it. Preload enhances data availability, boosting efficiency and reducing wait times. This is vital for AI training with large datasets as it prepares all necessary files in advance, saving GPU time and cutting costs. Combining preload with our blob filter feature allows customers to access specific files in a container or sub-directory, offering extensive flexibility and optimizing GPU cycles. - -To enable preload with file-cache mode, use `--preload` parameter. Below is a sample command for reference: -``` -blobfuse2 mount --preload /mnt/blobfuse_mnt --tmp-path=/home/temp_path -``` -/mnt/blobfuse_mnt is where the blob data can be accessed, and /home/temp_path serves as the cache for the Blobfuse mount. - - -Preloading blob data makes the mount read-only and prevents file eviction. To access updated files, unmount and remount the volume. Newly added files can still be accessed by reading them. If blob filter is used along with preload, only the filtered files are pre-loaded and accessible via the Blobfuse mount. - -### Considerations when using preload -- Enabling preload makes the Blobfuse mount read-only. -- All file-caching options in CLI and config file are ignored except for the temporary path setting. -- Ensure enough disk space for all or filtered contents in the container; insufficient space may cause partial loading and block new file access until manual deletion from the local cache. -- Accessing a file immediately after mounting prioritizes it for download while preloading continues in the background. -- Blobfuse logs show preload status and disk warnings. -- Blobfuse mount refreshes preloaded or opened files only if they are manually deleted from the local cache and reopened. -- Blobs added to the Storage container after preload are not automatically downloaded by Blobfuse but can be accessed by reading. - - -## Blob Filter -- In case of read-only mount, user can configure a filter to restrict what all blobs a mount can see or operate on. -- Blobfuse supports filters based on - - Name - - Size - - Last modified time - - File extension -- Blob Name based filter - - Supported operations are "=" and "!=" - - Name shall be a valid regex expression - - e.g. ```filter=name=^mine[0-1]\\d{3}.*``` -- Size based filter - - Supported operations are "<=", ">=", "!=", "<", ">" and "=" - - Size shall be provided in bytes - - e.g. ```filter=size > 1000``` -- Last Modified Date based filter - - Supported operations are "<=", ">=", "<", ">" and "=" - - Date shall be provided in RFC1123 Format e.g. "Mon, 24 Jan 1982 13:00:00 UTC" - - e.g. ```filter=modtime>Mon, 24 Jan 1982 13:00:00 UTC``` -- File Extension based filter - - Supported operations are "=" and "!=" - - Extension can be supplied as string. Do not include "." in the filter - - e.g. ```--filter=format=pdf``` -- Multiple filters can be combined using '&&' and '||' operator as well, however precedence using '()' is not supported yet. - - e.g. ```--filter=name=^testfil.* && size>130000000``` - -## Using Private Endpoints with HNS-Enabled Storage Accounts - -When accessing an HNS (Hierarchical Namespace) enabled Azure Storage account behind private endpoints, it is crucial to create **two separate private endpoints** to ensure proper connectivity: - -1. **Private Endpoint for DFS** - - Target: `privatelink.dfs.core.windows.net` - - This endpoint is necessary for accessing the Data Lake Storage Gen2 (HNS) functionality. - -2. **Private Endpoint for Blob** - - Target: `privatelink.blob.core.windows.net` - - This endpoint is necessary for accessing Blob Storage operations. - -### Why Both Endpoints Are Required - -HNS-enabled storage accounts utilize separate endpoints for Blob and DFS operations: -- The DFS endpoint (`dfs.core.windows.net`) is used for namespace-related operations like directory and file management. -- The Blob endpoint (`blob.core.windows.net`) is used for operations like streaming data to and from blobs. - -## Frequently Asked Questions -- How do I generate a SAS with permissions for rename? -az cli has a command to generate a sas token. Open a command prompt and make sure you are logged in to az cli. Run the following command and the sas token will be displayed in the command prompt. -az storage container generate-sas --account-name --account-key -n --permissions dlrwac --start --expiry -- Why do I get EINVAL on opening a file with WRONLY or APPEND flags? -To improve performance, Blobfuse2 by default enables writeback caching, which can produce unexpected behavior for files opened with WRONLY or APPEND flags, so Blobfuse2 returns EINVAL on open of a file with those flags. Either use disable-writeback-caching to turn off writeback caching (can potentially result in degraded performance) or ignore-open-flags (replace WRONLY with RDWR and ignore APPEND) based on your workload. -- How to mount blobfuse2 inside a container? -Refer to 'docker' folder in this repo. It contains a sample 'Dockerfile'. If you wish to create your own container image, try 'buildandruncontainer.sh' script, it will create a container image and launch the container using current environment variables holding your storage account credentials. -- Why am I not able to see the updated contents of file(s), which were updated through means other than Blobfuse2 mount? -If your use-case involves updating/uploading file(s) through other means and you wish to see the updated contents on Blobfuse2 mount then you need to disable kernel page-cache. `-o direct_io` CLI parameter is the option you need to use while mounting. Along with this, set `file-cache-timeout=0` and all other libfuse caching parameters should also be set to 0. User shall be aware that disabling kernel cache can result into more calls to Azure Storage which will have cost and performance implications. - -## Un-Supported File system operations -- mkfifo : fifo creation is not supported by blobfuse2 and this will result in "function not implemented" error -- chown : Change of ownership is not supported by Azure Storage hence Blobfuse2 does not support this. -- Creation of device files or pipes is not supported by Blobfuse2. -- Blobfuse2 does not support extended-attributes (x-attrs) operations -- Blobfuse2 does not support lseek() operation on directory handles. No error is thrown but it will not work as expected. - -## Un-Supported Scenarios -- Blobfuse2 does not support overlapping mount paths. While running multiple instances of Blobfuse2 make sure each instance has a unique and non-overlapping mount point. -- Blobfuse2 does not support co-existance with NFS on same mount path. Behaviour in this case is undefined. -- For block blob accounts, where data is uploaded through other means, Blobfuse2 expects special directory marker files to exist in container. In absence of this - few file operations might not work. For e.g. if you have a blob 'A/B/c.txt' then special marker files shall exists for 'A' and 'A/B', otherwise opening of 'A/B/c.txt' will fail. - Once a 'ls' operation is done on these directories 'A' and 'A/B' you will be able to open 'A/B/c.txt' as well. Possible workaround to resolve this from your container is to either - - create the directory marker files manually through portal or run 'mkdir' command for 'A' and 'A/B' from blobfuse. Refer [me](https://github.com/Azure/azure-storage-fuse/issues/866) - for details on this. - -## Limitations -- In case of BlockBlob accounts, ACLs are not supported by Azure Storage so Blobfuse2 will by default return success for 'chmod' operation. However it will work fine for Gen2 (DataLake) accounts. -- When Blobfuse2 is mounted on a container, SYS_ADMIN privileges are required for it to interact with the fuse driver. If container is created without the privilege, mount will fail. Sample command to spawn a docker container is - - `docker run -it --rm --cap-add=SYS_ADMIN --device=/dev/fuse --security-opt apparmor:unconfined ` -- In case of `mount all` system may limit on number of containers you can mount in parallel (when you go above 100 containers). To increase this system limit use below command - `echo 256 | sudo tee /proc/sys/fs/inotify/max_user_instances` -- Refer [this](#limitations-in-block-cache) for block-cache limitations. - -### Syslog security warning -By default, Blobfuse2 will log to syslog. The default settings will, in some cases, log relevant file paths to syslog. -If this is sensitive information, turn off logging or set log-level to LOG_ERR. - +## Limitations and known issues with BlobFuse +Refer the [page](https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-known-issues) for limitations and known issues. +For troubleshooting common issues, refer this [page](https://learn.microsoft.com/azure/storage/blobs/blobfuse2-troubleshooting). ## License This project is licensed under MIT. From d87a3c20c6e1cc9ca79f8bf67af3d6d8fade021a Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Wed, 11 Mar 2026 09:06:26 +0530 Subject: [PATCH 18/95] Add RMAN backup workload to nightly pipeline (#2138) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> --- .../rman-backup-simulation.yml | 66 ++ azure-pipeline-templates/rman-backup.yml | 69 +++ blobfuse2-nightly.yaml | 98 ++- test/scripts/rman_backup.sh | 574 ++++++++++++++++++ test/scripts/rman_backup_simulation.sh | 311 ++++++++++ 5 files changed, 1116 insertions(+), 2 deletions(-) create mode 100644 azure-pipeline-templates/rman-backup-simulation.yml create mode 100644 azure-pipeline-templates/rman-backup.yml create mode 100755 test/scripts/rman_backup.sh create mode 100755 test/scripts/rman_backup_simulation.sh diff --git a/azure-pipeline-templates/rman-backup-simulation.yml b/azure-pipeline-templates/rman-backup-simulation.yml new file mode 100644 index 000000000..49e6a5577 --- /dev/null +++ b/azure-pipeline-templates/rman-backup-simulation.yml @@ -0,0 +1,66 @@ +# Simulate Oracle RMAN backup workloads on blobfuse2 mount +# to verify data integrity with various database file sizes. +# +parameters: + - name: config_file + type: string + - name: cache_mode + type: string + - name: account_name + type: string + - name: account_key + type: string + - name: account_type + type: string + - name: verbose_log + type: boolean + default: false + +steps: + # Generate config file for file cache + - ${{ if eq(parameters.cache_mode, 'file_cache') }}: + - script: | + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} + cat ${{ parameters.config_file }} + displayName: 'Create Config File for File Cache' + env: + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.account_type }} + VERBOSE_LOG: ${{ parameters.verbose_log }} + continueOnError: false + + # Generate Config file for block cache + - ${{ if eq(parameters.cache_mode, 'block_cache') }}: + - script: | + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} + cat ${{ parameters.config_file }} + displayName: 'Create Config File for Block Cache' + env: + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.account_type }} + VERBOSE_LOG: ${{ parameters.verbose_log }} + + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.cache_mode }} + mountStep: + script: | + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 + + # Run RMAN backup simulation with various database file sizes + - script: + $(WORK_DIR)/test/scripts/rman_backup_simulation.sh $(MOUNT_DIR) $(ROOT_DIR) 10M,100M,1G,10G + displayName: 'RMAN Backup Simulation' + timeoutInMinutes: 120 + + - script: | + tail -n 5000 blobfuse2-logs.txt + displayName: 'View Logs' + condition: failed() + + - script: | + cat $(WORK_DIR)/*.trace + displayName: 'View stack trace' + condition: failed() diff --git a/azure-pipeline-templates/rman-backup.yml b/azure-pipeline-templates/rman-backup.yml new file mode 100644 index 000000000..3d5a46528 --- /dev/null +++ b/azure-pipeline-templates/rman-backup.yml @@ -0,0 +1,69 @@ +# Perform actual Oracle RMAN backup workloads on blobfuse2 mount +# to verify data integrity with various database file sizes. +# Requires Oracle XE to be available on the agent (installed if missing). +# +parameters: + - name: config_file + type: string + - name: cache_mode + type: string + - name: account_name + type: string + - name: account_key + type: string + - name: account_type + type: string + - name: verbose_log + type: boolean + default: false + +steps: + # Generate config file for file cache + - ${{ if eq(parameters.cache_mode, 'file_cache') }}: + - script: | + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} + cat ${{ parameters.config_file }} + displayName: 'Create Config File for File Cache' + env: + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.account_type }} + VERBOSE_LOG: ${{ parameters.verbose_log }} + continueOnError: false + + # Generate Config file for block cache + - ${{ if eq(parameters.cache_mode, 'block_cache') }}: + - script: | + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} + cat ${{ parameters.config_file }} + displayName: 'Create Config File for Block Cache' + env: + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.account_type }} + VERBOSE_LOG: ${{ parameters.verbose_log }} + + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.cache_mode }} + mountStep: + script: | + # Enable allow_other in fuse.conf so the oracle user can access the FUSE mount + sudo sed -i 's/^#\s*user_allow_other/user_allow_other/' /etc/fuse.conf + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 -o allow_other + + # Run actual RMAN backup tests with various database file sizes + - script: | + $(WORK_DIR)/test/scripts/rman_backup.sh $(MOUNT_DIR) $(ROOT_DIR) 10M,100M,1G,10G + displayName: 'RMAN Backup with Oracle XE' + timeoutInMinutes: 180 + + - script: | + tail -n 5000 blobfuse2-logs.txt + displayName: 'View Logs' + condition: failed() + + - script: | + cat $(WORK_DIR)/*.trace + displayName: 'View stack trace' + condition: failed() diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 78d20bef4..81a0732cd 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -41,6 +41,11 @@ parameters: type: boolean default: true + - name: rman_backup_test + displayName: 'Run RMAN backup simulation to check data integrity' + type: boolean + default: true + - name: linux_git_test displayName: 'Compile Linux Source and Run git clone Tests' type: string @@ -1085,7 +1090,7 @@ stages: jobs: # Ubuntu Tests - job: Set_1 - timeoutInMinutes: 60 + timeoutInMinutes: 360 strategy: matrix: Ubuntu-20: @@ -1161,7 +1166,7 @@ stages: jobs: # Ubuntu Tests - job: Set_1 - timeoutInMinutes: 60 + timeoutInMinutes: 180 strategy: matrix: Ubuntu-20: @@ -1232,6 +1237,95 @@ stages: unmount: true delete_containers: true + - ${{ if eq(parameters.rman_backup_test, true) }}: + - stage: RmanBackupTests + dependsOn: [] + jobs: + # Oracle Linux - native distro for Oracle XE and RMAN + - job: Set_1 + timeoutInMinutes: 360 + strategy: + matrix: + Oracle-8.1: + AgentName: 'blobfuse-oracle81' + distro: 'oracle' + distro_version: 'Oracle-8.1' + containerName: "test-cnt-ora-81-rman" + tags: 'fuse3' + + pool: + name: "blobfuse-oracle-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: NightlyBlobFuse + - name: MOUNT_DIR + value: '$(Pipeline.Workspace)/blob_mnt' + - name: TEMP_DIR + value: '$(Pipeline.Workspace)/blobfuse2_tmp' + - name: BLOBFUSE2_CFG + value: '$(Pipeline.Workspace)/blobfuse2.yaml' + - name: BLOBFUSE2_ADLS_CFG + value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + - name: skipComponentGovernanceDetection + value: true + - name: GOPATH + value: '$(Pipeline.Workspace)/go' + - name: ROOT_DIR + value: '$(Agent.TempDirectory)' + - name: WORK_DIR + value: '$(Build.SourcesDirectory)' + + steps: + # ------------------------------------------------------- + # Pull and build the code + - template: 'azure-pipeline-templates/build.yml' + parameters: + skip_ut: true + + - template: 'azure-pipeline-templates/rman-backup-simulation.yml' + parameters: + cache_mode: file_cache + config_file: $(BLOBFUSE2_CFG) + account_name: $(BF2_BLK_ACC_NAME) + account_key: $(BF2_BLK_ACC_KEY) + account_type: block + verbose_log: ${{ parameters.verbose_log }} + + - template: 'azure-pipeline-templates/rman-backup.yml' + parameters: + cache_mode: file_cache + config_file: $(BLOBFUSE2_CFG) + account_name: $(BF2_BLK_ACC_NAME) + account_key: $(BF2_BLK_ACC_KEY) + account_type: block + verbose_log: ${{ parameters.verbose_log }} + + # Disabling this test for block_cache due to known issues. + # - template: 'azure-pipeline-templates/rman-backup-simulation.yml' + # parameters: + # cache_mode: block_cache + # config_file: $(BLOBFUSE2_CFG) + # account_name: $(BF2_BLK_ACC_NAME) + # account_key: $(BF2_BLK_ACC_KEY) + # account_type: block + # verbose_log: ${{ parameters.verbose_log }} + + # - template: 'azure-pipeline-templates/rman-backup.yml' + # parameters: + # cache_mode: block_cache + # config_file: $(BLOBFUSE2_CFG) + # account_name: $(BF2_BLK_ACC_NAME) + # account_key: $(BF2_BLK_ACC_KEY) + # account_type: block + # verbose_log: ${{ parameters.verbose_log }} + + - template: 'azure-pipeline-templates/cleanup.yml' + parameters: + unmount: true + delete_containers: false + - ${{ if ne(parameters.linux_git_test, 'none') }}: - stage: CompileLinux_GitClone dependsOn: [] diff --git a/test/scripts/rman_backup.sh b/test/scripts/rman_backup.sh new file mode 100755 index 000000000..15061c831 --- /dev/null +++ b/test/scripts/rman_backup.sh @@ -0,0 +1,574 @@ +#!/bin/bash + +# rman_backup.sh +# Performs actual Oracle RMAN backup workloads on a FUSE-mounted filesystem. +# Installs Oracle XE, creates tablespaces with datafiles of various sizes, +# performs full and incremental backups using the rman utility, and validates +# data integrity. +# +# Usage: ./rman_backup.sh /path/to/mountpoint [sizes] +# sizes: comma-separated list of datafile sizes (default: 10M,100M,1G,10G) + +# Note: -e is intentionally omitted because the script uses explicit $? checks +# and graceful skip logic (exit 0 when Oracle is unavailable). +set -uo pipefail + +MOUNT_POINT="$1" +DATA_DIR="${2:-}" +SIZES="${3:-10M,100M,1G,10G}" + +if [ -z "$MOUNT_POINT" ]; then + echo "Usage: $0 /path/to/mountpoint [sizes]" + echo " sizes: comma-separated list (e.g., 10M,100M,1G,10G)" + exit 1 +fi + +if [ ! -d "$MOUNT_POINT" ]; then + echo "Error: Directory $MOUNT_POINT does not exist." + exit 1 +fi + +if [ -z "$DATA_DIR" ]; then + echo "Usage: $0 /path/to/mountpoint /path/to/data-dir [sizes]" + echo " data-dir: directory with sufficient space for Oracle database files (>=4.5GB)" + exit 1 +fi + +if [ ! -d "$DATA_DIR" ]; then + echo "Error: Data directory $DATA_DIR does not exist." + exit 1 +fi + +# Colors +GREEN='\033[0;32m' +RED='\033[0;31m' +CYAN='\033[0;36m' +NC='\033[0m' + +BACKUP_BASE="$MOUNT_POINT/rman_backup" +PASSED=0 +FAILED=0 + +IFS=',' read -ra SIZE_ARRAY <<< "$SIZES" + +# ============================================================================== +# setup_oracle_env: Set Oracle environment variables +# Detects ORACLE_HOME from common locations if not already set. +# ============================================================================== +setup_oracle_env() { + # Try to source Oracle's environment profile if available + if [ -f /opt/oracle/product/21c/dbhomeXE/bin/oracle_env.sh ]; then + . /opt/oracle/product/21c/dbhomeXE/bin/oracle_env.sh 2>/dev/null || true + elif [ -f /etc/profile.d/oracle-xe-21c.sh ]; then + . /etc/profile.d/oracle-xe-21c.sh 2>/dev/null || true + fi + + # Detect ORACLE_HOME from known paths if not set + if [ -z "${ORACLE_HOME:-}" ]; then + for candidate in /opt/oracle/product/21c/dbhomeXE /u01/app/oracle/product/21c/dbhomeXE; do + if [ -d "$candidate" ]; then + export ORACLE_HOME="$candidate" + break + fi + done + fi + + export ORACLE_HOME="${ORACLE_HOME:-/opt/oracle/product/21c/dbhomeXE}" + export ORACLE_SID="${ORACLE_SID:-XE}" + export PATH="$ORACLE_HOME/bin:$PATH" + export LD_LIBRARY_PATH="$ORACLE_HOME/lib:${LD_LIBRARY_PATH:-}" + export NLS_LANG="AMERICAN_AMERICA.AL32UTF8" +} + +# Set Oracle environment early so cleanup and all functions have access +setup_oracle_env + +cleanup() { + echo "Cleaning up..." + + # Drop test tablespaces if Oracle is running + if pgrep -x "ora_pmon_XE" > /dev/null 2>&1; then + for SIZE in "${SIZE_ARRAY[@]}"; do + ts_name="BFUSE_${SIZE}" + run_sqlplus " + BEGIN + EXECUTE IMMEDIATE 'ALTER TABLESPACE ${ts_name} OFFLINE'; + EXCEPTION WHEN OTHERS THEN NULL; + END; + / + BEGIN + EXECUTE IMMEDIATE 'DROP TABLESPACE ${ts_name} INCLUDING CONTENTS AND DATAFILES'; + EXCEPTION WHEN OTHERS THEN NULL; + END; + / + " 2>/dev/null || true + done + fi + + rm -rf "$BACKUP_BASE" +} + +trap cleanup EXIT + +# ============================================================================== +# run_sqlplus: Execute SQL statement via sqlplus as the oracle OS user. +# Uses 'sudo -u oracle' to ensure OS authentication ('/ as sysdba') succeeds, +# since the oracle user is in the dba group by default. +# ============================================================================== +run_sqlplus() { + local sql="$1" + local script + script=$(printf 'SET HEADING OFF FEEDBACK OFF PAGESIZE 0 LINESIZE 200\nWHENEVER SQLERROR EXIT SQL.SQLCODE\n%s\nEXIT;\n' "$sql") + sudo -E -u oracle \ + ORACLE_HOME="${ORACLE_HOME}" \ + ORACLE_SID="${ORACLE_SID}" \ + PATH="${ORACLE_HOME}/bin:${PATH}" \ + LD_LIBRARY_PATH="${ORACLE_HOME}/lib" \ + NLS_LANG="AMERICAN_AMERICA.AL32UTF8" \ + bash -c 'echo "$1" | sqlplus -s "/ as sysdba"' _ "$script" +} + +# ============================================================================== +# run_rman: Execute RMAN commands as the oracle OS user. +# ============================================================================== +run_rman() { + local commands="$1" + local script + script=$(printf '%s\nEXIT;\n' "$commands") + sudo -E -u oracle \ + ORACLE_HOME="${ORACLE_HOME}" \ + ORACLE_SID="${ORACLE_SID}" \ + PATH="${ORACLE_HOME}/bin:${PATH}" \ + LD_LIBRARY_PATH="${ORACLE_HOME}/lib" \ + bash -c 'echo "$1" | rman target /' _ "$script" +} + +# ============================================================================== +# install_oracle_xe: Install Oracle XE if not already installed +# ============================================================================== +install_oracle_xe() { + if command -v sqlplus &> /dev/null && pgrep -x "ora_pmon_XE" > /dev/null 2>&1; then + echo -e "${CYAN}Oracle XE is already installed and running${NC}" + return 0 + fi + + echo -e "${CYAN}Installing Oracle XE...${NC}" + + # Install prerequisites (Oracle Linux uses yum) + sudo yum install -y libaio bc wget unzip + + # Download and install Oracle XE natively via RPM (Oracle Linux) + if ! command -v sqlplus &> /dev/null; then + ORACLE_XE_RPM="oracle-database-xe-21c-1.0-1.ol8.x86_64.rpm" + if [ ! -f "$DATA_DIR/${ORACLE_XE_RPM}" ]; then + echo -e "${CYAN}Downloading Oracle XE...${NC}" + wget -q "https://download.oracle.com/otn-pub/otn_software/db-express/${ORACLE_XE_RPM}" \ + -O "$DATA_DIR/${ORACLE_XE_RPM}" || { + echo -e "${RED}Failed to download Oracle XE.${NC}" + echo -e "${RED}[SKIP] Oracle XE not available for download. Skipping actual RMAN tests.${NC}" + return 1 + } + fi + + echo -e "${CYAN}Installing Oracle XE RPM...${NC}" + sudo yum localinstall -y "$DATA_DIR/${ORACLE_XE_RPM}" || { + echo -e "${RED}[SKIP] Failed to install Oracle XE RPM. Skipping actual RMAN tests.${NC}" + return 1 + } + fi + + # Set Oracle data location to DATA_DIR to avoid insufficient space on /opt/oracle + if [ -f /etc/sysconfig/oracle-xe-21c.conf ]; then + if grep -q "^DBFILE_DEST=" /etc/sysconfig/oracle-xe-21c.conf; then + sudo sed -i "s|^DBFILE_DEST=.*|DBFILE_DEST=${DATA_DIR//|/\\|}|" /etc/sysconfig/oracle-xe-21c.conf + else + echo "DBFILE_DEST=${DATA_DIR}" | sudo tee -a /etc/sysconfig/oracle-xe-21c.conf + fi + fi + + # Configure Oracle XE with password from environment (test-only default) + local ora_pwd="${ORACLE_XE_PASSWORD:-Oracle123}" + echo -e "${CYAN}Configuring Oracle XE...${NC}" + printf '%s\n%s\n' "$ora_pwd" "$ora_pwd" | sudo /etc/init.d/oracle-xe-21c configure + if [ $? -ne 0 ]; then + echo -e "${RED}[SKIP] Failed to configure Oracle XE. Skipping actual RMAN tests.${NC}" + return 1 + fi + + # Re-detect Oracle environment after installation + setup_oracle_env + return 0 +} + +# ============================================================================== +# create_tablespace: Create a tablespace with a datafile of the given size +# ============================================================================== +create_tablespace() { + local size="$1" + local ts_name="BFUSE_${size}" + local df_dir="${DATA_DIR}/oradata/XE/bfuse_test" + + echo -e "${CYAN}Creating tablespace ${ts_name} with ${size} datafile...${NC}" + + # Create datafile directory + sudo mkdir -p "$df_dir" + sudo chown oracle:oinstall "$df_dir" + + # Drop if exists + run_sqlplus " + BEGIN + EXECUTE IMMEDIATE 'ALTER TABLESPACE ${ts_name} OFFLINE'; + EXCEPTION WHEN OTHERS THEN NULL; + END; + / + BEGIN + EXECUTE IMMEDIATE 'DROP TABLESPACE ${ts_name} INCLUDING CONTENTS AND DATAFILES'; + EXCEPTION WHEN OTHERS THEN NULL; + END; + / + " 2>/dev/null || true + + # Create tablespace + run_sqlplus "CREATE TABLESPACE ${ts_name} DATAFILE '${df_dir}/${ts_name}.dbf' SIZE ${size} AUTOEXTEND OFF;" + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] Failed to create tablespace ${ts_name}${NC}" + return 1 + fi + + # Populate with data to ensure the datafile is fully written + run_sqlplus " + CREATE TABLE test_data_${size} TABLESPACE ${ts_name} AS + SELECT level AS id, + dbms_random.string('A', 100) AS data, + SYSDATE AS created_at + FROM dual + CONNECT BY level <= 10000; + " || true + + echo -e "${GREEN}Tablespace ${ts_name} created successfully${NC}" + return 0 +} + +# ============================================================================== +# rman_full_backup: Perform actual RMAN full backup to FUSE mount +# ============================================================================== +rman_full_backup() { + local size="$1" + local ts_name="BFUSE_${size}" + local backup_dir="$BACKUP_BASE/full_${size}" + + echo -e "${CYAN}------------------------------------------------------------${NC}" + echo -e "${CYAN}RMAN Full Backup: tablespace=${ts_name} dest=${backup_dir}${NC}" + echo -e "${CYAN}------------------------------------------------------------${NC}" + + mkdir -p "$backup_dir" + chmod 777 "$backup_dir" + + # Perform RMAN full backup of the tablespace to blobfuse2 mount + run_rman " + CONFIGURE CONTROLFILE AUTOBACKUP ON; + CONFIGURE CONTROLFILE AUTOBACKUP FORMAT FOR DEVICE TYPE DISK TO '${backup_dir}/ctrl_%F'; + BACKUP AS BACKUPSET + TABLESPACE ${ts_name} + FORMAT '${backup_dir}/full_%U.bkp' + TAG 'FULL_${ts_name}'; + " + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] RMAN full backup failed for ${ts_name}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Verify backup files exist on the FUSE mount + local bkp_count + bkp_count=$(find "$backup_dir" -name "full_*.bkp" -type f | wc -l) + if [ "$bkp_count" -eq 0 ]; then + echo -e "${RED}[FAIL] No backup files found in ${backup_dir}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Drop page cache to force read from FUSE + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' 2>/dev/null + + # Perform RESTORE VALIDATE to confirm backup can be restored. + # This does a dry-run restore: RMAN reads every backup piece from the + # FUSE mount, validates every Oracle block's checksum, and confirms + # recoverability — without writing anything to disk. + echo -e "${CYAN}Running RESTORE VALIDATE...${NC}" + run_rman " + RESTORE TABLESPACE ${ts_name} VALIDATE; + " + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] RMAN RESTORE VALIDATE failed for full backup of ${ts_name}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + echo -e "${GREEN}[PASS] RMAN full backup integrity verified: ${ts_name} (${bkp_count} backup pieces)${NC}" + PASSED=$((PASSED + 1)) + return 0 +} + +# ============================================================================== +# rman_incremental_backup: Perform RMAN incremental backup +# ============================================================================== +rman_incremental_backup() { + local size="$1" + local ts_name="BFUSE_${size}" + local backup_dir="$BACKUP_BASE/incr_${size}" + + echo -e "${CYAN}------------------------------------------------------------${NC}" + echo -e "${CYAN}RMAN Incremental Backup: tablespace=${ts_name} dest=${backup_dir}${NC}" + echo -e "${CYAN}------------------------------------------------------------${NC}" + + mkdir -p "$backup_dir" + chmod 777 "$backup_dir" + + # Level 0 incremental backup (base) + echo -e "${CYAN}Creating Level 0 incremental backup...${NC}" + run_rman " + BACKUP INCREMENTAL LEVEL 0 + TABLESPACE ${ts_name} + FORMAT '${backup_dir}/incr0_%U.bkp' + TAG 'INCR0_${ts_name}'; + " + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] RMAN level 0 incremental backup failed for ${ts_name}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Make some changes to the tablespace + echo -e "${CYAN}Modifying data in tablespace...${NC}" + run_sqlplus " + INSERT INTO test_data_${size} + SELECT level + 10000 AS id, + dbms_random.string('A', 100) AS data, + SYSDATE AS created_at + FROM dual + CONNECT BY level <= 5000; + COMMIT; + " || true + + # Level 1 incremental backup + echo -e "${CYAN}Creating Level 1 incremental backup...${NC}" + run_rman " + BACKUP INCREMENTAL LEVEL 1 + TABLESPACE ${ts_name} + FORMAT '${backup_dir}/incr1_%U.bkp' + TAG 'INCR1_${ts_name}'; + " + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] RMAN level 1 incremental backup failed for ${ts_name}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Drop page cache + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' 2>/dev/null + + # Validate level 0 backup: dry-run restore reads every backup piece and + # confirms recoverability without writing anything. + echo -e "${CYAN}Validating Level 0 backup...${NC}" + run_rman " + RESTORE TABLESPACE ${ts_name} VALIDATE; + " + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] RMAN VALIDATE failed for level 0 backup of ${ts_name}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Validate level 1 backup + echo -e "${CYAN}Validating Level 1 backup...${NC}" + run_rman " + RESTORE TABLESPACE ${ts_name} VALIDATE; + " + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] RMAN VALIDATE failed for level 1 backup of ${ts_name}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + echo -e "${GREEN}[PASS] RMAN incremental backup integrity verified: ${ts_name}${NC}" + PASSED=$((PASSED + 1)) + return 0 +} + +# ============================================================================== +# rman_backup_and_restore_verify: Full backup, restore, and compare +# ============================================================================== +rman_backup_and_restore_verify() { + local size="$1" + local ts_name="BFUSE_${size}" + local backup_dir="$BACKUP_BASE/verify_${size}" + local df_dir="${DATA_DIR}/oradata/XE/bfuse_test" + local datafile="${df_dir}/${ts_name}.dbf" + + echo -e "${CYAN}------------------------------------------------------------${NC}" + echo -e "${CYAN}RMAN Backup & Restore Verify: tablespace=${ts_name}${NC}" + echo -e "${CYAN}------------------------------------------------------------${NC}" + + mkdir -p "$backup_dir" + chmod 777 "$backup_dir" + + # Write the datafile to the FUSE mount as an RMAN copy. + echo -e "${CYAN}Creating datafile copy on FUSE mount...${NC}" + run_rman " + BACKUP AS COPY + DATAFILE '${datafile}' + FORMAT '${backup_dir}/${ts_name}_copy.dbf'; + " + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] RMAN backup as copy failed for ${ts_name}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Verify the copy file was actually written to the FUSE mount + if [ ! -f "${backup_dir}/${ts_name}_copy.dbf" ]; then + echo -e "${RED}[FAIL] Backup copy not found on FUSE mount: ${backup_dir}/${ts_name}_copy.dbf${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Drop page cache to force subsequent reads from FUSE + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' 2>/dev/null + + # Use RMAN RESTORE VALIDATE to verify the copy on the FUSE mount. + # This is a dry-run restore: RMAN reads the copy from the FUSE mount, + # validates every Oracle block's checksum, and confirms the tablespace + # can be recovered — without writing anything to disk. + echo -e "${CYAN}Validating copy with RMAN RESTORE VALIDATE...${NC}" + run_rman " + RESTORE TABLESPACE ${ts_name} VALIDATE; + " + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] RMAN RESTORE VALIDATE failed for ${ts_name}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + echo -e "${GREEN}[PASS] RMAN backup copy integrity verified: ${ts_name}${NC}" + PASSED=$((PASSED + 1)) + return 0 +} + +# ============================================================================== +# Main test execution +# ============================================================================== +echo "============================================================" +echo "Starting Actual RMAN Backup Tests on $MOUNT_POINT" +echo "Database file sizes: ${SIZES}" +echo "============================================================" + +# Verify Oracle XE is available (check both PATH and ORACLE_HOME/bin) +if ! command -v rman &> /dev/null && [ ! -x "${ORACLE_HOME}/bin/rman" ]; then + echo -e "${CYAN}RMAN utility not found, attempting Oracle XE installation...${NC}" + install_oracle_xe + if [ $? -ne 0 ]; then + echo -e "${RED}Oracle XE installation failed. Skipping actual RMAN tests.${NC}" + echo -e "${CYAN}Note: Simulation-based RMAN tests are still run separately.${NC}" + exit 0 + fi +fi + +# Verify the database is running +if ! pgrep -x "ora_pmon_XE" > /dev/null 2>&1; then + echo -e "${CYAN}Starting Oracle XE database...${NC}" + sudo /etc/init.d/oracle-xe-21c start 2>/dev/null || { + echo -e "${RED}Failed to start Oracle XE. Skipping actual RMAN tests.${NC}" + exit 0 + } +fi + +# Verify OS authentication works before proceeding +echo -e "${CYAN}Verifying Oracle OS authentication...${NC}" +run_sqlplus "SELECT 'AUTH_OK' FROM dual;" +if [ $? -ne 0 ]; then + echo -e "${RED}Oracle OS authentication failed. Skipping actual RMAN tests.${NC}" + echo -e "${CYAN}Ensure the 'oracle' user exists and the database is running.${NC}" + exit 0 +fi + +# Enable ARCHIVELOG mode if not already enabled. +# RMAN cannot backup or copy active datafiles in NOARCHIVELOG mode (ORA-19602). +echo -e "${CYAN}Checking ARCHIVELOG mode...${NC}" +ARCHIVELOG_STATUS=$(run_sqlplus "SELECT LOG_MODE FROM V\$DATABASE;") +if echo "$ARCHIVELOG_STATUS" | grep -q "NOARCHIVELOG"; then + echo -e "${CYAN}Enabling ARCHIVELOG mode (requires restart)...${NC}" + run_sqlplus "SHUTDOWN IMMEDIATE;" || { + echo -e "${RED}Failed to shutdown database for ARCHIVELOG switch. Skipping.${NC}" + exit 0 + } + run_sqlplus "STARTUP MOUNT;" || { + echo -e "${RED}Failed to start database in mount mode. Attempting full startup...${NC}" + run_sqlplus "STARTUP;" + exit 0 + } + run_sqlplus "ALTER DATABASE ARCHIVELOG;" || { + echo -e "${RED}Failed to enable ARCHIVELOG mode. Opening database and skipping.${NC}" + run_sqlplus "ALTER DATABASE OPEN;" + exit 0 + } + run_sqlplus "ALTER DATABASE OPEN;" || { + echo -e "${RED}Failed to open database after ARCHIVELOG switch.${NC}" + exit 0 + } + echo -e "${GREEN}ARCHIVELOG mode enabled${NC}" +else + echo -e "${CYAN}ARCHIVELOG mode is already enabled${NC}" +fi + +mkdir -p "$BACKUP_BASE" +# chmod 777 is needed because blobfuse2 FUSE mounts do not support POSIX +# ownership/ACLs, and the oracle user (via sudo -u oracle) must write here. +chmod 777 "$BACKUP_BASE" + +# Configure RMAN defaults +echo -e "${CYAN}Configuring RMAN defaults...${NC}" +run_rman " + CONFIGURE RETENTION POLICY TO NONE; + CONFIGURE BACKUP OPTIMIZATION ON; + CONFIGURE DEVICE TYPE DISK PARALLELISM 1; + CONFIGURE DEFAULT DEVICE TYPE TO DISK; +" + +for SIZE in "${SIZE_ARRAY[@]}"; do + echo "" + echo "============================================================" + echo "Testing with database file size: ${SIZE}" + echo "============================================================" + + # Create tablespace with the specified datafile size + create_tablespace "$SIZE" + if [ $? -ne 0 ]; then + FAILED=$((FAILED + 1)) + continue + fi + + # Test 1: Full RMAN backup with integrity validation + rman_full_backup "$SIZE" + + # Test 2: Incremental RMAN backup (level 0 + level 1) with validation + rman_incremental_backup "$SIZE" + + # Test 3: RMAN backup as copy and MD5 verification + rman_backup_and_restore_verify "$SIZE" +done + +echo "" +echo "============================================================" +echo "Actual RMAN Backup Tests Complete" +echo " Passed: $PASSED" +echo " Failed: $FAILED" +echo "============================================================" + +if [ $FAILED -gt 0 ]; then + echo -e "${RED}Some tests FAILED!${NC}" + exit 1 +fi + +echo -e "${GREEN}All actual RMAN backup tests passed!${NC}" +exit 0 diff --git a/test/scripts/rman_backup_simulation.sh b/test/scripts/rman_backup_simulation.sh new file mode 100755 index 000000000..b9f83dab8 --- /dev/null +++ b/test/scripts/rman_backup_simulation.sh @@ -0,0 +1,311 @@ +#!/bin/bash + +# rman_backup_simulation.sh +# Simulates Oracle RMAN backup workloads on a FUSE-mounted filesystem. +# Creates database files of various sizes, performs full and incremental +# backup simulations, and validates data integrity via MD5 checksums. +# +# Usage: ./rman_backup_simulation.sh /path/to/mountpoint /path/to/data_dir [sizes] +# data_dir: directory where source data files will be created +# sizes: comma-separated list of file sizes (default: 10M,100M,1G,10G) + +MOUNT_POINT="$1" +DATA_DIR="$2" +SIZES="${3:-10M,100M,1G,10G}" + +if [ -z "$MOUNT_POINT" ] || [ -z "$DATA_DIR" ]; then + echo "Usage: $0 /path/to/mountpoint /path/to/data_dir [sizes]" + echo " data_dir: directory where source data files will be created" + echo " sizes: comma-separated list (e.g., 10M,100M,1G,10G)" + exit 1 +fi + +if [ ! -d "$MOUNT_POINT" ]; then + echo "Error: Directory $MOUNT_POINT does not exist." + exit 1 +fi + +# Colors +GREEN='\033[0;32m' +RED='\033[0;31m' +CYAN='\033[0;36m' +NC='\033[0m' + +BACKUP_DIR="$MOUNT_POINT/rman_backup" +SOURCE_DIR="$DATA_DIR/rman_source_$$" +PASSED=0 +FAILED=0 + +cleanup() { + echo "Cleaning up..." + rm -rf "$SOURCE_DIR" + rm -rf "$BACKUP_DIR" +} + +trap cleanup EXIT + +mkdir -p "$SOURCE_DIR" +mkdir -p "$BACKUP_DIR" + +IFS=',' read -ra SIZE_ARRAY <<< "$SIZES" + +# ============================================================================== +# generate_datafile: Create a source file simulating an Oracle datafile +# ============================================================================== +generate_datafile() { + local size="$1" + local file="$SOURCE_DIR/datafile_${size}.dbf" + + echo -e "${CYAN}Generating source datafile: ${size}${NC}" >&2 + + # Use dd with appropriate block size for the file size + case "$size" in + 10M) dd if=/dev/urandom of="$file" bs=1M count=10 status=none ;; + 100M) dd if=/dev/urandom of="$file" bs=1M count=100 status=none ;; + 1G) dd if=/dev/urandom of="$file" bs=1M count=1024 status=none ;; + 10G) dd if=/dev/urandom of="$file" bs=1M count=10240 status=none ;; + *) dd if=/dev/urandom of="$file" bs=1M count=10 status=none ;; + esac + + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] Failed to generate datafile ${size}${NC}" >&2 + return 1 + fi + + echo "$file" + return 0 +} + +# ============================================================================== +# full_backup: Simulate RMAN full backup (sequential write with large blocks) +# RMAN typically writes backup pieces using 256K or 1M block sizes. +# ============================================================================== +full_backup() { + local src_file="$1" + local size="$2" + local bs="$3" + local backup_file="$BACKUP_DIR/full_${size}_bs${bs}.bkp" + + echo -e "${CYAN}------------------------------------------------------------${NC}" + echo -e "${CYAN}Full Backup: size=${size} block_size=${bs}${NC}" + echo -e "${CYAN}------------------------------------------------------------${NC}" + + # Simulate RMAN full backup: sequential write with specified block size + dd if="$src_file" of="$backup_file" bs="$bs" conv=fsync status=none + if [ $? -ne 0 ]; then + echo -e "${RED}[FAIL] Full backup write failed for ${size} bs=${bs}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Drop page cache to force read from FUSE + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' 2>/dev/null + + # Verify integrity + local src_md5 + local bkp_md5 + src_md5=$(md5sum "$src_file" | awk '{print $1}') + bkp_md5=$(md5sum "$backup_file" | awk '{print $1}') + + if [ "$src_md5" == "$bkp_md5" ]; then + echo -e "${GREEN}[PASS] Full backup integrity verified: ${size} bs=${bs}${NC}" + echo " MD5: $src_md5" + PASSED=$((PASSED + 1)) + else + echo -e "${RED}[FAIL] Full backup integrity MISMATCH: ${size} bs=${bs}${NC}" + echo " Source MD5: $src_md5" + echo " Backup MD5: $bkp_md5" + FAILED=$((FAILED + 1)) + return 1 + fi + + rm -f "$backup_file" + return 0 +} + +# ============================================================================== +# incremental_backup: Simulate RMAN incremental backup +# Writes partial blocks at various offsets to simulate changed-block tracking. +# ============================================================================== +incremental_backup() { + local src_file="$1" + local size="$2" + local backup_file="$BACKUP_DIR/incr_${size}.bkp" + + echo -e "${CYAN}------------------------------------------------------------${NC}" + echo -e "${CYAN}Incremental Backup Simulation: size=${size}${NC}" + echo -e "${CYAN}------------------------------------------------------------${NC}" + + # First, create a full copy as the base + cp "$src_file" "$backup_file" + sync + + # Simulate incremental changes: overwrite specific blocks in the source + local file_size_bytes + file_size_bytes=$(stat -c%s "$src_file") + + # Write changed blocks at 3 different offsets (beginning, middle, near end) + local offsets=("0" "$((file_size_bytes / 3))" "$((file_size_bytes * 2 / 3))") + local block_size=1048576 # 1M blocks + + for offset in "${offsets[@]}"; do + dd if=/dev/urandom of="$src_file" bs="$block_size" count=1 seek=$((offset / block_size)) conv=notrunc status=none + done + + # Apply the incremental changes to the backup (simulate merge) + for offset in "${offsets[@]}"; do + dd if="$src_file" of="$backup_file" bs="$block_size" count=1 \ + skip=$((offset / block_size)) seek=$((offset / block_size)) conv=notrunc,fsync status=none + done + + # Drop page cache + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' 2>/dev/null + + # Verify the merged backup matches the modified source + local src_md5 + local bkp_md5 + src_md5=$(md5sum "$src_file" | awk '{print $1}') + bkp_md5=$(md5sum "$backup_file" | awk '{print $1}') + + if [ "$src_md5" == "$bkp_md5" ]; then + echo -e "${GREEN}[PASS] Incremental backup integrity verified: ${size}${NC}" + echo " MD5: $src_md5" + PASSED=$((PASSED + 1)) + else + echo -e "${RED}[FAIL] Incremental backup integrity MISMATCH: ${size}${NC}" + echo " Source MD5: $src_md5" + echo " Backup MD5: $bkp_md5" + FAILED=$((FAILED + 1)) + return 1 + fi + + rm -f "$backup_file" + return 0 +} + +# ============================================================================== +# multi_channel_backup: Simulate RMAN multi-channel backup +# Writes multiple backup pieces in parallel, then verifies each. +# ============================================================================== +multi_channel_backup() { + local src_file="$1" + local size="$2" + local channels=4 + local file_size_bytes + file_size_bytes=$(stat -c%s "$src_file") + local piece_size=$((file_size_bytes / channels)) + + echo -e "${CYAN}------------------------------------------------------------${NC}" + echo -e "${CYAN}Multi-Channel Backup: size=${size} channels=${channels}${NC}" + echo -e "${CYAN}------------------------------------------------------------${NC}" + + # Split the source into pieces and write them in parallel + local pids=() + for i in $(seq 0 $((channels - 1))); do + local piece_file="$BACKUP_DIR/piece_${size}_ch${i}.bkp" + local skip_blocks=$((i * piece_size / 1048576)) + local count_blocks=$((piece_size / 1048576)) + # Last channel gets any remaining bytes + if [ $i -eq $((channels - 1)) ]; then + count_blocks=$(( (file_size_bytes - i * piece_size) / 1048576 )) + fi + dd if="$src_file" of="$piece_file" bs=1M skip="$skip_blocks" count="$count_blocks" conv=fsync status=none & + pids+=($!) + done + + # Wait for all channels to complete + local write_failed=0 + for pid in "${pids[@]}"; do + wait "$pid" || write_failed=1 + done + + if [ $write_failed -eq 1 ]; then + echo -e "${RED}[FAIL] Multi-channel backup write failed: ${size}${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi + + # Drop page cache + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' 2>/dev/null + + # Reassemble and verify + local reassembled="$BACKUP_DIR/reassembled_${size}.bkp" + > "$reassembled" + for i in $(seq 0 $((channels - 1))); do + cat "$BACKUP_DIR/piece_${size}_ch${i}.bkp" >> "$reassembled" + done + + local src_md5 + local asm_md5 + src_md5=$(md5sum "$src_file" | awk '{print $1}') + asm_md5=$(md5sum "$reassembled" | awk '{print $1}') + + if [ "$src_md5" == "$asm_md5" ]; then + echo -e "${GREEN}[PASS] Multi-channel backup integrity verified: ${size}${NC}" + echo " MD5: $src_md5" + PASSED=$((PASSED + 1)) + else + echo -e "${RED}[FAIL] Multi-channel backup integrity MISMATCH: ${size}${NC}" + echo " Source MD5: $src_md5" + echo " Reassembled MD5: $asm_md5" + FAILED=$((FAILED + 1)) + return 1 + fi + + rm -f "$BACKUP_DIR/piece_${size}_"*.bkp "$reassembled" + return 0 +} + +# ============================================================================== +# Main test execution +# ============================================================================== +echo "============================================================" +echo "Starting RMAN Backup Simulation on $MOUNT_POINT" +echo "Database file sizes: ${SIZES}" +echo "============================================================" + +for SIZE in "${SIZE_ARRAY[@]}"; do + echo "" + echo "============================================================" + echo "Testing with database file size: ${SIZE}" + echo "============================================================" + + # Generate source datafile + SRC_FILE=$(generate_datafile "$SIZE") + if [ $? -ne 0 ]; then + FAILED=$((FAILED + 1)) + continue + fi + + # Test 1: Full backup with 256K block size (typical RMAN default) + full_backup "$SRC_FILE" "$SIZE" "256K" + + # Test 2: Full backup with 1M block size (RMAN MAXPIECESIZE-style large writes) + full_backup "$SRC_FILE" "$SIZE" "1M" + + # Test 3: Incremental backup simulation + incremental_backup "$SRC_FILE" "$SIZE" + + # Test 4: Multi-channel parallel backup (skip for smallest size) + if [ "$SIZE" != "10M" ]; then + multi_channel_backup "$SRC_FILE" "$SIZE" + fi + + # Clean up source file after testing this size + rm -f "$SRC_FILE" +done + +echo "" +echo "============================================================" +echo "RMAN Backup Simulation Complete" +echo " Passed: $PASSED" +echo " Failed: $FAILED" +echo "============================================================" + +if [ $FAILED -gt 0 ]; then + echo -e "${RED}Some tests FAILED!${NC}" + exit 1 +fi + +echo -e "${GREEN}All RMAN backup simulation tests passed!${NC}" +exit 0 From 2bdfeda14751bb0cbef59a314f54e168ca7d22ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 10:38:12 +0530 Subject: [PATCH 19/95] Bump github.com/montanaflynn/stats from 0.7.1 to 0.8.2 (#2157) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b5e5f26e2..9dba18a50 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/golang/mock v1.6.0 - github.com/montanaflynn/stats v0.7.1 + github.com/montanaflynn/stats v0.8.2 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/radovskyb/watcher v1.0.7 github.com/sevlyar/go-daemon v0.1.6 diff --git a/go.sum b/go.sum index cb22c5f50..587b12f4b 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= -github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/montanaflynn/stats v0.8.2 h1:52wnefTJnPI5FoHif1DQh2soKRw0yYs+4AVyvtcZCH0= +github.com/montanaflynn/stats v0.8.2/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= From 506c4fc60abc57dc53896bad03428d8abb7f0393 Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Thu, 12 Mar 2026 17:39:53 +0530 Subject: [PATCH 20/95] Parallel FlushFile calls in file cache (#2136) --- CHANGELOG.md | 5 +- component/file_cache/file_cache.go | 219 +++++++++--------- component/file_cache/file_cache_test.go | 281 ++++++++++++++++++++++++ 3 files changed, 404 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b583b316..80f43dcdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,9 @@ - Add rate limit functionality for ingress bandwidth (bytes downloaded per second) and operations per second ([PR #2093](https://github.com/Azure/azure-storage-fuse/pull/2093)) **Bug Fixes** -- Fix panic while reading an archived blob using file-cache ([PR #2127](https://github.com/Azure/azure-storage-fuse/pull/2127)] -- Ensure logger is properly deinitialized on mount failures ([PR #2125](https://github.com/Azure/azure-storage-fuse/pull/2125)] +- Fix panic while reading an archived blob using file cache ([PR #2127](https://github.com/Azure/azure-storage-fuse/pull/2127)) +- Ensure logger is properly deinitialized on mount failures ([PR #2125](https://github.com/Azure/azure-storage-fuse/pull/2125)) +- Fix InvalidBlockList error caused by parallel flush file calls in file cache ([PR #2136](https://github.com/Azure/azure-storage-fuse/pull/2136)) ## 2.5.2 (2026-01-19) **Features** diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index 0eb87b880..05d67a33d 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -1066,39 +1066,28 @@ func (fc *FileCache) OpenFile(options internal.OpenFileOptions) (*handlemap.Hand // ReleaseFile: Flush the file and invalidate it from the cache. func (fc *FileCache) ReleaseFile(options internal.ReleaseFileOptions) error { - // Lock the file so that while close is in progress no one can open the file again - flock := fc.fileLocks.Get(options.Handle.Path) - flock.Lock() + log.Trace("FileCache::ReleaseFile : name=%s, handle=%d", options.Handle.Path, options.Handle.ID) // Async close is called so schedule the upload and return here fc.fileCloseOpt.Add(1) - - if !fc.lazyWrite { - // Sync close is called so wait till the upload completes - return fc.releaseFileInternal(options, flock) - } - - go fc.releaseFileInternal(options, flock) //nolint - return nil -} - -// releaseFileInternal: Actual handling of the close file goes here -func (fc *FileCache) releaseFileInternal(options internal.ReleaseFileOptions, flock *common.LockMapItem) error { - log.Trace("FileCache::releaseFileInternal : name=%s, handle=%d", options.Handle.Path, options.Handle.ID) - - // Lock is acquired by ReleaseFile, at end of this method we need to unlock - // If its async call file shall be locked till the upload completes. - defer flock.Unlock() defer fc.fileCloseOpt.Done() localPath := filepath.Join(fc.tmpPath, options.Handle.Path) + // FlushFile takes lock on the given file to ensure that while close is in progress no one can open the file again err := fc.FlushFile(internal.FlushFileOptions{Handle: options.Handle, CloseInProgress: true}) //nolint if err != nil { log.Err("FileCache::releaseFileInternal : failed to flush file %s", options.Handle.Path) return err } + // Take lock and decrement the handle count for this file. + // We need to take lock here to make sure that cache policy does not delete + // the file while we are in the middle of closing it and updating handle count. + flock := fc.fileLocks.Get(options.Handle.Path) + flock.Lock() + defer flock.Unlock() + f := options.Handle.GetFileObject() if f == nil { log.Err("FileCache::releaseFileInternal : error [missing fd in handle object] %s", options.Handle.Path) @@ -1110,6 +1099,7 @@ func (fc *FileCache) releaseFileInternal(options internal.ReleaseFileOptions, fl log.Err("FileCache::releaseFileInternal : error closing file %s(%d) [%s]", options.Handle.Path, int(f.Fd()), err.Error()) return err } + flock.Dec() // If it is an fsync op then purge the file @@ -1268,6 +1258,14 @@ func (fc *FileCache) SyncFile(options internal.SyncFileOptions) error { // } // FlushFile: Flush the local file to storage +// FlushFile can be called concurrently by multiple libfuse_flush calls for different file descriptors +// (e.g. due to dup(), dup2() or fork()) for the same file. Without serialization, each fd would independently +// call CopyFromFile (which does PutBlock + PutBlockList), leading to InvalidBlockList errors when +// concurrent PutBlockList calls race against each other. +// To prevent this, we acquire a per-file lock before uploading the file. +// +// Fast-path: If the handle is not dirty, or if lazyWrite defers the upload, we return immediately +// without acquiring the lock to avoid unnecessary contention on the hot flush path. func (fc *FileCache) FlushFile(options internal.FlushFileOptions) error { //defer exectime.StatTimeCurrentBlock("FileCache::FlushFile")() log.Trace("FileCache::FlushFile : handle=%d, path=%s", options.Handle.ID, options.Handle.Path) @@ -1275,102 +1273,125 @@ func (fc *FileCache) FlushFile(options internal.FlushFileOptions) error { // The file should already be in the cache since CreateFile/OpenFile was called before and a shared lock was acquired. localPath := filepath.Join(fc.tmpPath, options.Handle.Path) fc.policy.CacheValid(localPath) - // if our handle is dirty then that means we wrote to the file - if options.Handle.Dirty() { - if fc.lazyWrite && !options.CloseInProgress { - // As lazy-write is enable, upload will be scheduled when file is closed. - log.Info("FileCache::FlushFile : %s will be flushed when handle %d is closed", options.Handle.Path, options.Handle.ID) - return nil - } - f := options.Handle.GetFileObject() - if f == nil { - log.Err("FileCache::FlushFile : error [couldn't find fd in handle] %s", options.Handle.Path) - return syscall.EBADF - } + // Fast-path: skip the lock entirely when no upload will happen. + // libfuse calls flush on every close() of a file descriptor, even when nothing was written, + // so this avoids contention for the common read-only / clean-handle case. + if !options.Handle.Dirty() { + return nil + } - // Flush all data to disk that has been buffered by the kernel. - // We cannot close the incoming handle since the user called flush, note close and flush can be called on the same handle multiple times. - // To ensure the data is flushed to disk before writing to storage, we duplicate the handle and close that handle. - // f.fsync() is another option but dup+close does it quickly compared to sync - dupFd, err := syscall.Dup(int(f.Fd())) - if err != nil { - log.Err("FileCache::FlushFile : error [couldn't duplicate the fd] %s", options.Handle.Path) - return syscall.EIO - } + // Fast-path: if lazy-write is enabled and file is not being closed, the upload is deferred + // to file close time. No lock needed since we are not touching storage. + if fc.lazyWrite && !options.CloseInProgress { + log.Info("FileCache::FlushFile : %s will be flushed when handle %d is closed", options.Handle.Path, options.Handle.ID) + return nil + } - err = syscall.Close(dupFd) - if err != nil { - log.Err("FileCache::FlushFile : error [unable to close duplicate fd] %s", options.Handle.Path) - return syscall.EIO - } + // Acquire per-file lock to serialize concurrent upload operations for the same file. + // Multiple libfuse_flush calls (from dup'd fds) can arrive in parallel and each would + // attempt PutBlock + PutBlockList, causing InvalidBlockList errors on the storage side. + flock := fc.fileLocks.Get(options.Handle.Path) + flock.Lock() + defer flock.Unlock() + + // Re-check Dirty() after acquiring the lock. + // Another goroutine that held the lock before us may have already uploaded the file + // and cleared the dirty flag. Uploading again would be redundant (and the second + // PutBlockList could race with the next write sequence, producing InvalidBlockList). + if !options.Handle.Dirty() { + log.Debug("FileCache::FlushFile : %s handle %d is no longer dirty after acquiring lock, skipping upload", + options.Handle.Path, options.Handle.ID) + return nil + } - // Write to storage - // Create a new handle for the SDK to use to upload (read local file) - // The local handle can still be used for read and write. - var orgMode fs.FileMode - modeChanged := false + f := options.Handle.GetFileObject() + if f == nil { + log.Err("FileCache::FlushFile : error [couldn't find fd in handle] %s", options.Handle.Path) + return syscall.EBADF + } - uploadHandle, err := os.Open(localPath) - if err != nil { - if os.IsPermission(err) { - info, _ := os.Stat(localPath) - orgMode = info.Mode() - newMode := orgMode | 0444 - err = os.Chmod(localPath, newMode) - if err == nil { - modeChanged = true - uploadHandle, err = os.Open(localPath) - log.Info("FileCache::FlushFile : read mode added to file %s", options.Handle.Path) - } - } + // Flush all data to disk that has been buffered by the kernel. + // We cannot close the incoming handle since the user called flush, note close and flush can be called on the same handle multiple times. + // To ensure the data is flushed to disk before writing to storage, we duplicate the handle and close that handle. + // f.fsync() is another option but dup+close does it quickly compared to sync + dupFd, err := syscall.Dup(int(f.Fd())) + if err != nil { + log.Err("FileCache::FlushFile : error [couldn't duplicate the fd] %s", options.Handle.Path) + return syscall.EIO + } - if err != nil { - log.Err("FileCache::FlushFile : error [unable to open upload handle] %s [%s]", options.Handle.Path, err.Error()) - return err - } - } - err = fc.NextComponent().CopyFromFile( - internal.CopyFromFileOptions{ - Name: options.Handle.Path, - File: uploadHandle, - }) + err = syscall.Close(dupFd) + if err != nil { + log.Err("FileCache::FlushFile : error [unable to close duplicate fd] %s", options.Handle.Path) + return syscall.EIO + } - uploadHandle.Close() + // Write to storage + // Create a new handle for the SDK to use to upload (read local file) + // The local handle can still be used for read and write. + var orgMode fs.FileMode + modeChanged := false - if modeChanged { - err1 := os.Chmod(localPath, orgMode) - if err1 != nil { - log.Err("FileCache::FlushFile : Failed to remove read mode from file %s [%s]", options.Handle.Path, err1.Error()) + uploadHandle, err := os.Open(localPath) + if err != nil { + if os.IsPermission(err) { + info, _ := os.Stat(localPath) + orgMode = info.Mode() + newMode := orgMode | 0444 + err = os.Chmod(localPath, newMode) + if err == nil { + modeChanged = true + uploadHandle, err = os.Open(localPath) + log.Info("FileCache::FlushFile : read mode added to file %s", options.Handle.Path) } } if err != nil { - log.Err("FileCache::FlushFile : %s upload failed [%s]", options.Handle.Path, err.Error()) + log.Err("FileCache::FlushFile : error [unable to open upload handle] %s [%s]", options.Handle.Path, err.Error()) return err } + } + err = fc.NextComponent().CopyFromFile( + internal.CopyFromFileOptions{ + Name: options.Handle.Path, + File: uploadHandle, + }) - options.Handle.Flags.Clear(handlemap.HandleFlagDirty) + uploadHandle.Close() - // If chmod was done on the file before it was uploaded to container then setting up mode would have been missed - // Such file names are added to this map and here post upload we try to set the mode correctly - _, found := fc.missedChmodList.Load(options.Handle.Path) - if found { - // If file is found in map it means last chmod was missed on this - // Delete the entry from map so that any further flush do not try to update the mode again - fc.missedChmodList.Delete(options.Handle.Path) + if modeChanged { + err1 := os.Chmod(localPath, orgMode) + if err1 != nil { + log.Err("FileCache::FlushFile : Failed to remove read mode from file %s [%s]", options.Handle.Path, err1.Error()) + } + } - // When chmod on container was missed, local file was updated with correct mode - // Here take the mode from local cache and update the container accordingly - localPath := filepath.Join(fc.tmpPath, options.Handle.Path) - info, err := os.Lstat(localPath) - if err == nil { - err = fc.Chmod(internal.ChmodOptions{Name: options.Handle.Path, Mode: info.Mode()}) - if err != nil { - // chmod was missed earlier for this file and doing it now also - // resulted in error so ignore this one and proceed for flush handling - log.Err("FileCache::FlushFile : %s chmod failed [%s]", options.Handle.Path, err.Error()) - } + if err != nil { + log.Err("FileCache::FlushFile : %s upload failed [%s]", options.Handle.Path, err.Error()) + return err + } + + options.Handle.Flags.Clear(handlemap.HandleFlagDirty) + + // If chmod was done on the file before it was uploaded to container then setting up mode would have been missed + // Such file names are added to this map and here post upload we try to set the mode correctly + _, found := fc.missedChmodList.Load(options.Handle.Path) + if found { + // If file is found in map it means last chmod was missed on this + // Delete the entry from map so that any further flush do not try to update the mode again + fc.missedChmodList.Delete(options.Handle.Path) + + // When chmod on container was missed, local file was updated with correct mode + // Here take the mode from local cache and update the container accordingly + localPath := filepath.Join(fc.tmpPath, options.Handle.Path) + info, err := os.Lstat(localPath) + if err == nil { + err = fc.Chmod(internal.ChmodOptions{Name: options.Handle.Path, Mode: info.Mode()}) + if err != nil { + // chmod was missed earlier for this file and doing it now also + // resulted in error so ignore this one and proceed for flush handling + log.Err("FileCache::FlushFile : %s chmod failed [%s]", options.Handle.Path, err.Error()) } } } diff --git a/component/file_cache/file_cache_test.go b/component/file_cache/file_cache_test.go index 4df36aefc..c927023ce 100644 --- a/component/file_cache/file_cache_test.go +++ b/component/file_cache/file_cache_test.go @@ -45,6 +45,8 @@ import ( "path/filepath" "strconv" "strings" + "sync" + "sync/atomic" "syscall" "testing" "time" @@ -70,6 +72,7 @@ type fileCacheTestSuite struct { loopback internal.Component cache_path string fake_storage_path string + configString string } func newLoopbackFS() internal.Component { @@ -117,6 +120,7 @@ func (suite *fileCacheTestSuite) SetupTest() { func (suite *fileCacheTestSuite) setupTestHelper(configuration string) { suite.assert = assert.New(suite.T()) + suite.configString = configuration err := config.ReadConfigFromReader(strings.NewReader(configuration)) suite.assert.NoError(err) @@ -1251,6 +1255,283 @@ func (suite *fileCacheTestSuite) TestFlushFileErrorBadFd() { suite.assert.EqualValues(syscall.EBADF, err) } +// setupMockFileCacheForFlush creates a file cache backed by a gomock NextComponent +// and returns (fileCache, mockComponent, cachePath, cleanup). +// The caller must defer cleanup() to stop the independent file cache and remove its temp dir. +// The suite-level loopback and fileCache are left running throughout and are not affected. +func (suite *fileCacheTestSuite) setupMockFileCacheForFlush(mockCtrl *gomock.Controller) (*FileCache, *internal.MockComponent, string, func()) { + mockComponent := internal.NewMockComponent(mockCtrl) + + randStr := randomString(8) + cachePath := filepath.Join(home_dir, "file_cache"+randStr) + cfg := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n timeout-sec: 0", cachePath) + + // Overwrite global config to configure the independent FileCache, then immediately + // restore the original config so the suite-level components are unaffected. + err := config.ReadConfigFromReader(strings.NewReader(cfg)) + suite.assert.NoError(err) + + fcComp := NewFileCacheComponent() + fcComp.SetNextComponent(mockComponent) + err = fcComp.Configure(true) + suite.assert.NoError(err) + + // Restore the suite's original config so subsequent suite-level operations work correctly. + err = config.ReadConfigFromReader(strings.NewReader(suite.configString)) + suite.assert.NoError(err) + + mockComponent.EXPECT().Start(gomock.Any()).Return(nil).Times(1) + mockComponent.EXPECT().Stop().Return(nil).Times(1) + + err = mockComponent.Start(context.Background()) + suite.assert.NoError(err) + err = fcComp.Start(context.Background()) + suite.assert.NoError(err) + + fc := fcComp.(*FileCache) + + cleanup := func() { + _ = fc.Stop() + _ = mockComponent.Stop() + os.RemoveAll(cachePath) + } + + return fc, mockComponent, cachePath, cleanup +} + +// TestFlushFileConcurrent verifies that concurrent FlushFile calls for the same file +// are serialized by the per-file lock. +// +// The mock NextComponent's CopyFromFile tracks the number of concurrent in-flight +// uploads via an atomic counter and sleeps briefly to widen the race window. +// If two uploads ever overlap (counter > 1), the mock returns an error that mimics +// the Azure Storage InvalidBlockList failure. +// With the per-file lock this can never happen; without it, the overlapping calls +// would cause the test to fail. +func (suite *fileCacheTestSuite) TestFlushFileConcurrent() { + defer suite.cleanupTest() + mockCtrl := gomock.NewController(suite.T()) + defer mockCtrl.Finish() + + fc, mockComponent, cachePath, cleanup := suite.setupMockFileCacheForFlush(mockCtrl) + defer cleanup() + + // Create a real local file in the cache (createEmptyFile=false means no NextComponent call). + path := "concurrent_flush.txt" + localPath := filepath.Join(cachePath, path) + err := os.MkdirAll(filepath.Dir(localPath), 0755) + suite.assert.NoError(err) + f, err := os.Create(localPath) + suite.assert.NoError(err) + _, err = f.WriteString("concurrent flush test data") + suite.assert.NoError(err) + + handle := handlemap.NewHandle(path) + handle.UnixFD = uint64(f.Fd()) + handle.SetFileObject(f) + handle.Flags.Set(handlemap.HandleFlagDirty) + fc.fileLocks.Get(path).Inc() // simulate an open handle + + // Track concurrent in-flight CopyFromFile calls. + var inFlight atomic.Int32 + var maxInFlight atomic.Int32 + + // The mock CopyFromFile blocks briefly so that without serialization, + // multiple goroutines would be inside it at the same time. + mockComponent.EXPECT(). + CopyFromFile(gomock.Any()). + DoAndReturn(func(opts internal.CopyFromFileOptions) error { + cur := inFlight.Add(1) + // Record the high-water mark of concurrent callers. + for { + old := maxInFlight.Load() + if cur <= old || maxInFlight.CompareAndSwap(old, cur) { + break + } + } + + // Hold the "upload" open long enough for other goroutines to arrive. + time.Sleep(50 * time.Millisecond) + + inFlight.Add(-1) + + if cur > 1 { + // This is what Azure Storage would return when two PutBlockList calls race. + return fmt.Errorf("InvalidBlockList: concurrent upload detected (%d in-flight)", cur) + } + return nil + }). + AnyTimes() + + // Launch concurrent flushes (simulating multiple libfuse_flush calls from dup'd fds). + concurrency := 5 + var wg sync.WaitGroup + errs := make([]error, concurrency) + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + go func(idx int) { + defer wg.Done() + // Re-dirty the handle before each flush so that the fast-path dirty check + // doesn't skip the upload. In the real world each fd's close sets dirty via + // libfuse_flush (fileHandle.dirty != 0). + handle.Flags.Set(handlemap.HandleFlagDirty) + errs[idx] = fc.FlushFile(internal.FlushFileOptions{Handle: handle}) + }(i) + } + wg.Wait() + + for i, e := range errs { + suite.assert.NoError(e, "FlushFile goroutine %d returned error: %v", i, e) + } + + // The per-file lock must have serialized all uploads; at no point should more + // than one CopyFromFile have been in-flight simultaneously. + suite.assert.EqualValues(1, maxInFlight.Load(), + "expected max 1 concurrent CopyFromFile, got %d — uploads were not serialized", maxInFlight.Load()) + + f.Close() +} + +// TestFlushFileConcurrentWithRelease verifies that a concurrent FlushFile call +// (from libfuse_flush) and ReleaseFile (close) are properly serialized so that +// CopyFromFile never runs in parallel. +func (suite *fileCacheTestSuite) TestFlushFileConcurrentWithRelease() { + defer suite.cleanupTest() + mockCtrl := gomock.NewController(suite.T()) + defer mockCtrl.Finish() + + fc, mockComponent, cachePath, cleanup := suite.setupMockFileCacheForFlush(mockCtrl) + defer cleanup() + + path := "concurrent_flush_release.txt" + localPath := filepath.Join(cachePath, path) + err := os.MkdirAll(filepath.Dir(localPath), 0755) + suite.assert.NoError(err) + f, err := os.Create(localPath) + suite.assert.NoError(err) + _, err = f.WriteString("concurrent flush release test data") + suite.assert.NoError(err) + + handle := handlemap.NewHandle(path) + handle.UnixFD = uint64(f.Fd()) + handle.SetFileObject(f) + handle.Flags.Set(handlemap.HandleFlagDirty) + fc.fileLocks.Get(path).Inc() // simulate an open handle + + var inFlight atomic.Int32 + var maxInFlight atomic.Int32 + + mockComponent.EXPECT(). + CopyFromFile(gomock.Any()). + DoAndReturn(func(opts internal.CopyFromFileOptions) error { + cur := inFlight.Add(1) + for { + old := maxInFlight.Load() + if cur <= old || maxInFlight.CompareAndSwap(old, cur) { + break + } + } + time.Sleep(50 * time.Millisecond) + inFlight.Add(-1) + if cur > 1 { + return fmt.Errorf("InvalidBlockList: concurrent upload detected (%d in-flight)", cur) + } + return nil + }). + AnyTimes() + + // Start a concurrent flush to simulate libfuse_flush from a dup'd fd. + var wg sync.WaitGroup + var flushErr error + wg.Add(1) + go func() { + defer wg.Done() + flushErr = fc.FlushFile(internal.FlushFileOptions{Handle: handle}) + }() + + // ReleaseFile on the same handle races with the flush above. + // releaseFileInternal calls FlushFile internally, so both paths compete. + releaseErr := fc.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + + wg.Wait() + suite.assert.NoError(flushErr, "concurrent FlushFile should not fail") + suite.assert.NoError(releaseErr, "ReleaseFile should not fail") + suite.assert.EqualValues(1, maxInFlight.Load(), + "expected max 1 concurrent CopyFromFile, got %d — uploads were not serialized", maxInFlight.Load()) +} + +// TestFlushFileSyncFileConcurrent verifies that SyncFile (which calls FlushFile internally +// when syncToFlush is enabled) and a direct FlushFile call are serialized so that +// CopyFromFile never runs concurrently. +func (suite *fileCacheTestSuite) TestFlushFileSyncFileConcurrent() { + defer suite.cleanupTest() + mockCtrl := gomock.NewController(suite.T()) + defer mockCtrl.Finish() + + fc, mockComponent, cachePath, cleanup := suite.setupMockFileCacheForFlush(mockCtrl) + defer cleanup() + fc.syncToFlush = true + + path := "sync_flush_concurrent.txt" + localPath := filepath.Join(cachePath, path) + err := os.MkdirAll(filepath.Dir(localPath), 0755) + suite.assert.NoError(err) + f, err := os.Create(localPath) + suite.assert.NoError(err) + _, err = f.WriteString("sync flush concurrent test data") + suite.assert.NoError(err) + + handle := handlemap.NewHandle(path) + handle.UnixFD = uint64(f.Fd()) + handle.SetFileObject(f) + handle.Flags.Set(handlemap.HandleFlagDirty) + fc.fileLocks.Get(path).Inc() // simulate an open handle + + var inFlight atomic.Int32 + var maxInFlight atomic.Int32 + + mockComponent.EXPECT(). + CopyFromFile(gomock.Any()). + DoAndReturn(func(opts internal.CopyFromFileOptions) error { + cur := inFlight.Add(1) + for { + old := maxInFlight.Load() + if cur <= old || maxInFlight.CompareAndSwap(old, cur) { + break + } + } + time.Sleep(50 * time.Millisecond) + inFlight.Add(-1) + if cur > 1 { + return fmt.Errorf("InvalidBlockList: concurrent upload detected (%d in-flight)", cur) + } + return nil + }). + AnyTimes() + + // Launch concurrent SyncFile (calls FlushFile internally) and direct FlushFile. + var wg sync.WaitGroup + var syncErr, flushErr error + + wg.Add(2) + go func() { + defer wg.Done() + syncErr = fc.SyncFile(internal.SyncFileOptions{Handle: handle}) + }() + go func() { + defer wg.Done() + flushErr = fc.FlushFile(internal.FlushFileOptions{Handle: handle}) + }() + wg.Wait() + + suite.assert.NoError(syncErr, "SyncFile should not fail") + suite.assert.NoError(flushErr, "FlushFile should not fail") + suite.assert.EqualValues(1, maxInFlight.Load(), + "expected max 1 concurrent CopyFromFile, got %d — uploads were not serialized", maxInFlight.Load()) + + f.Close() +} + func (suite *fileCacheTestSuite) TestGetAttrCase1() { defer suite.cleanupTest() // Setup From 0bf9b9fd08c108cf05e1d11de2c0803454265e6f Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Thu, 12 Mar 2026 20:49:46 +0530 Subject: [PATCH 21/95] Updating version to 2.5.3 (#2158) --- common/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/types.go b/common/types.go index 745c2a32b..858a4c923 100644 --- a/common/types.go +++ b/common/types.go @@ -47,7 +47,7 @@ import ( // Standard config default values const ( - blobfuse2Version_ = "2.5.2" + blobfuse2Version_ = "2.5.3" DefaultMaxLogFileSize = 512 DefaultLogFileCount = 10 From f99e4575490575b37e5be1d9aca1ced78a2b139a Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Sun, 15 Mar 2026 13:49:00 +0530 Subject: [PATCH 22/95] Adding version files for warnings (#2159) --- release/blockedversions/1.1.1 | 0 release/blockedversions/2.3.0 | 0 release/securitywarnings/1.1.1 | 0 release/securitywarnings/2.2.0 | 0 release/securitywarnings/2.2.1 | 0 release/securitywarnings/2.3.0 | 0 release/securitywarnings/2.3.0~preview.1 | 0 release/securitywarnings/2.3.1~preview.1 | 0 8 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 release/blockedversions/1.1.1 create mode 100644 release/blockedversions/2.3.0 create mode 100644 release/securitywarnings/1.1.1 create mode 100644 release/securitywarnings/2.2.0 create mode 100644 release/securitywarnings/2.2.1 create mode 100644 release/securitywarnings/2.3.0 create mode 100644 release/securitywarnings/2.3.0~preview.1 create mode 100644 release/securitywarnings/2.3.1~preview.1 diff --git a/release/blockedversions/1.1.1 b/release/blockedversions/1.1.1 new file mode 100644 index 000000000..e69de29bb diff --git a/release/blockedversions/2.3.0 b/release/blockedversions/2.3.0 new file mode 100644 index 000000000..e69de29bb diff --git a/release/securitywarnings/1.1.1 b/release/securitywarnings/1.1.1 new file mode 100644 index 000000000..e69de29bb diff --git a/release/securitywarnings/2.2.0 b/release/securitywarnings/2.2.0 new file mode 100644 index 000000000..e69de29bb diff --git a/release/securitywarnings/2.2.1 b/release/securitywarnings/2.2.1 new file mode 100644 index 000000000..e69de29bb diff --git a/release/securitywarnings/2.3.0 b/release/securitywarnings/2.3.0 new file mode 100644 index 000000000..e69de29bb diff --git a/release/securitywarnings/2.3.0~preview.1 b/release/securitywarnings/2.3.0~preview.1 new file mode 100644 index 000000000..e69de29bb diff --git a/release/securitywarnings/2.3.1~preview.1 b/release/securitywarnings/2.3.1~preview.1 new file mode 100644 index 000000000..e69de29bb From 9f2f4bc74dbdce4adccd23dd95dde5eca7fa43b2 Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:11:55 +0530 Subject: [PATCH 23/95] Remote version check for updates and warnings (#2160) --- blobfuse2-release.yaml | 92 ---------------------- cmd/root.go | 126 +++++++++++++++++++++--------- cmd/root_test.go | 28 +++---- common/version.go | 20 ++++- component/azstorage/utils_test.go | 8 -- 5 files changed, 118 insertions(+), 156 deletions(-) diff --git a/blobfuse2-release.yaml b/blobfuse2-release.yaml index 70743b140..c2b677f66 100644 --- a/blobfuse2-release.yaml +++ b/blobfuse2-release.yaml @@ -1805,95 +1805,3 @@ stages: done < <(tail -n +3 ../packages.csv) workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - stage: UpdateLatestVersion - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: UpdateVersion - timeoutInMinutes: 120 - strategy: - matrix: - UBN20: - vmImage: 'ubuntu-20.04' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - depends: 'fuse3' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - checkout: none - - - ${{ if eq(parameters.update_version, true) }}: - - script: | - sudo apt-get install python3 -y - python3 --version - displayName: 'Installing Python' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2' - downloadPath: $(Build.ArtifactStagingDirectory) - - # install blobfuse2 - - script: | - sudo apt install fuse3 -y - cd $(Build.ArtifactStagingDirectory)/blobfuse2 - ls | grep -i ubuntu-20.04 - sudo apt-get install ./`ls | grep -i ubuntu-20.04` -y - blobfuse2 version - displayName: 'Installing blobfuse2' - - - script: | - mkdir -p ~/blob_mnt - chmod 777 ~/blob_mnt - mkdir -p ~/blobfusetemp - chmod 777 ~/blobfusetemp - blobfuse2 mount ~/blob_mnt --tmp-path=~/blobfusetemp --container-name="\$web" - sleep 2 - env: - AZURE_STORAGE_ACCOUNT: "blobfuse2" - AZURE_STORAGE_AUTH_TYPE: "msi" - AZURE_STORAGE_IDENTITY_CLIENT_ID: $(BLOBFUSE2_MSI_CIENT_ID) - displayName: 'Mount release container for version update' - - - script: | - cd ~/blob_mnt/release/latest/ - echo "Before cleanup" - ls -l - rm -rf * - echo "After cleanup" - ls -l - displayName: 'Delete older version file' - - - script: | - cd ~/blob_mnt/release/latest/ - touch `blobfuse2 version | cut -d " " -f3` - ls -l - displayName: 'Create new version file' - - - script: | - cd ~/blob_mnt/release/latest/ - version=`blobfuse2 version | cut -d " " -f3` - echo "Release version: " $version - echo -e "\n$version\n" > index.xml - echo "Printing index.xml" - cat index.xml - displayName: 'Update the XML file' - - - script: | - blobfuse2 unmount all - displayName: 'Unmount Blobfuse2' - diff --git a/cmd/root.go b/cmd/root.go index 209d10aad..aee6d72b2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,10 +34,10 @@ package cmd import ( + "encoding/json" "encoding/xml" "errors" "fmt" - "io" "net/http" "os" "slices" @@ -78,49 +78,83 @@ var rootCmd = &cobra.Command{ }, } -// check if the version file exists in the container +// check if the version file exists in the GitHub release directory func checkVersionExists(versionUrl string) bool { - resp, err := http.Get(versionUrl) + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: getTransport(), + } + + req, err := http.NewRequest("GET", versionUrl, nil) + if err != nil { + log.Err("checkVersionExists: error creating request [%s]", err.Error()) + return false + } + + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + + resp, err := client.Do(req) if err != nil { - log.Err("checkVersionExists: error getting version file from container [%s]", err.Error()) + log.Err("checkVersionExists: error getting version file from GitHub [%s]", err.Error()) return false } + defer resp.Body.Close() - return resp.StatusCode != 404 + // Treat only 2xx as "exists". 404 is a normal "does not exist" case. + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return true + } + if resp.StatusCode == http.StatusNotFound { + return false + } + + // For other status codes (e.g., 401, 403, 5xx), log and treat as unknown/non-existent. + log.Err("checkVersionExists: unexpected status code from GitHub [%d] for URL %s", resp.StatusCode, versionUrl) + return false } -// getRemoteVersion : From public container get the latest blobfuse version -func getRemoteVersion(req string) (string, error) { - resp, err := http.Get(req) +func getGitHubLatestRemoteVersion(apiEndpoint string) (*common.Version, error) { + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: getTransport(), + } + + // Get Request + req, err := http.NewRequest("GET", apiEndpoint, nil) if err != nil { - log.Err("getRemoteVersion: error listing version file from container [%s]", err.Error()) - return "", err + return nil, err } - body, err := io.ReadAll(resp.Body) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + req.Header.Set("User-Agent", "blobfuse2-version-check") + + resp, err := client.Do(req) if err != nil { - log.Err("getRemoteVersion: error reading body of response [%s]", err.Error()) - return "", err + log.Err("getGitHubLatestRemoteVersion: error in GitHub GET latest release [%s]", err.Error()) + return nil, err } + defer resp.Body.Close() - if len(body) > 50 { - log.Err("getRemoteVersion: something suspicious in the contents from remote version") - return "", fmt.Errorf("unable to get latest version") + if resp.StatusCode != 200 { + return nil, fmt.Errorf("getGitHubLatestRemoteVersion: error in GitHub GET latest release: %s", resp.Status) } - var versionList VersionFilesList - err = xml.Unmarshal(body, &versionList) - if err != nil { - log.Err("getRemoteVersion: error unmarshalling xml response [%s]", err.Error()) - return "", err + var release struct { // JSON response representation + TagName string `json:"tag_name"` } - if len(versionList.Version) < 5 || len(versionList.Version) > 20 { - return "", fmt.Errorf("unable to get latest version") + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&release) + if err != nil { + return nil, err } - versionName := versionList.Version - return versionName, nil + // Remove blobfuse2- prefix in TagName, convert str to Version + // Tag name example, blobfuse2-2.5.2 + versionStr := strings.TrimPrefix(release.TagName, "blobfuse2-") + return common.ParseVersion(versionStr) } // beginDetectNewVersion : Get latest release version and compare if user needs an upgrade or not @@ -130,8 +164,7 @@ func beginDetectNewVersion() chan any { go func() { defer close(completed) - latestVersionUrl := common.Blobfuse2ListContainerURL + "/latest/index.xml" - remoteVersion, err := getRemoteVersion(latestVersionUrl) + remote, err := getGitHubLatestRemoteVersion(common.GitHubLatestReleaseURL) if err != nil { log.Err("beginDetectNewVersion: error getting latest version [%s]", err.Error()) if strings.Contains(err.Error(), "no such host") { @@ -148,20 +181,13 @@ func beginDetectNewVersion() chan any { return } - remote, err := common.ParseVersion(remoteVersion) - if err != nil { - log.Err("beginDetectNewVersion: error parsing remoteVersion [%s]", err.Error()) - completed <- err.Error() - return - } - - warningsUrl := common.Blobfuse2ListContainerURL + "/securitywarnings/" + common.Blobfuse2Version + warningsUrl := common.GitHubReleaseContentsURL + "/securitywarnings/" + common.Blobfuse2Version hasWarnings := checkVersionExists(warningsUrl) if hasWarnings { // This version has known issues associated with it // Check whether the version has been blocked by the dev team or not. - blockedVersions := common.Blobfuse2ListContainerURL + "/blockedversions/" + common.Blobfuse2Version + blockedVersions := common.GitHubReleaseContentsURL + "/blockedversions/" + common.Blobfuse2Version isBlocked := checkVersionExists(blockedVersions) if isBlocked { @@ -181,9 +207,9 @@ func beginDetectNewVersion() chan any { if local.OlderThan(*remote) { executablePathSegments := strings.Split(strings.ReplaceAll(os.Args[0], "\\", "/"), "/") executableName := executablePathSegments[len(executablePathSegments)-1] - log.Info("beginDetectNewVersion: A new version of Blobfuse2 is available. Current Version=%s, Latest Version=%s", common.Blobfuse2Version, remoteVersion) - fmt.Fprintf(stderr, "*** "+executableName+": A new version [%s] is available. Consider upgrading to latest version for bug-fixes & new features. ***\n", remoteVersion) - log.Info("*** "+executableName+": A new version [%s] is available. Consider upgrading to latest version for bug-fixes & new features. ***\n", remoteVersion) + log.Info("beginDetectNewVersion: A new version of Blobfuse2 is available. Current Version=%s, Latest Version=%s", common.Blobfuse2Version, remote.String()) + fmt.Fprintf(stderr, "*** "+executableName+": A new version [%s] is available. Consider upgrading to latest version for bug-fixes & new features. ***\n", remote.String()) + log.Info("*** "+executableName+": A new version [%s] is available. Consider upgrading to latest version for bug-fixes & new features. ***\n", remote.String()) completed <- "A new version of Blobfuse2 is available" } @@ -274,6 +300,28 @@ func parseArgs(cmdArgs []string) []string { return args } +func getTransport() *http.Transport { + // Prefer cloning the default transport so we inherit standard + // proxy/TLS settings (including HTTP(S)_PROXY and NO_PROXY). + if dt, ok := http.DefaultTransport.(*http.Transport); ok { + cp := dt.Clone() + cp.MaxIdleConns = 10 + cp.IdleConnTimeout = 30 * time.Second + cp.DisableCompression = true // GitHub API responses are small + cp.DisableKeepAlives = false // Connections are reused + return cp + } + + // Fallback: construct a transport that at least respects proxy env vars. + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, // GitHub API responses are small + DisableKeepAlives: false, // Connections are reused + } +} + // Execute : Actual command execution starts from here func Execute() error { parsedArgs := parseArgs(os.Args) diff --git a/cmd/root_test.go b/cmd/root_test.go index 8959f3074..c0ead9085 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -91,56 +91,56 @@ func (suite *rootCmdSuite) TestCheckVersionExistsInvalidURL() { func (suite *rootCmdSuite) TestNoSecurityWarnings() { defer suite.cleanupTest() - warningsUrl := common.Blobfuse2ListContainerURL + "/securitywarnings/" + common.Blobfuse2Version + warningsUrl := common.GitHubReleaseContentsURL + "/securitywarnings/" + common.Blobfuse2Version found := checkVersionExists(warningsUrl) suite.assert.False(found) } func (suite *rootCmdSuite) TestSecurityWarnings() { defer suite.cleanupTest() - warningsUrl := common.Blobfuse2ListContainerURL + "/securitywarnings/" + "1.1.1" + warningsUrl := common.GitHubReleaseContentsURL + "/securitywarnings/" + "1.1.1" found := checkVersionExists(warningsUrl) suite.assert.True(found) } func (suite *rootCmdSuite) TestBlockedVersion() { defer suite.cleanupTest() - warningsUrl := common.Blobfuse2ListContainerURL + "/blockedversions/" + "1.1.1" + warningsUrl := common.GitHubReleaseContentsURL + "/blockedversions/" + "1.1.1" isBlocked := checkVersionExists(warningsUrl) suite.assert.True(isBlocked) } func (suite *rootCmdSuite) TestNonBlockedVersion() { defer suite.cleanupTest() - warningsUrl := common.Blobfuse2ListContainerURL + "/blockedversions/" + common.Blobfuse2Version + warningsUrl := common.GitHubReleaseContentsURL + "/blockedversions/" + common.Blobfuse2Version found := checkVersionExists(warningsUrl) suite.assert.False(found) } func (suite *rootCmdSuite) TestGetRemoteVersionInvalidURL() { defer suite.cleanupTest() - out, err := getRemoteVersion("abcd") - suite.assert.Empty(out) + out, err := getGitHubLatestRemoteVersion("abcd") + suite.assert.Nil(out) suite.assert.Error(err) } -func (suite *rootCmdSuite) TestGetRemoteVersionInvalidContainer() { +func (suite *rootCmdSuite) TestGetRemoteVersionInvalidRepo() { defer suite.cleanupTest() - latestVersionUrl := common.Blobfuse2ListContainerURL + "?restype=container&comp=list&prefix=latest1/" - out, err := getRemoteVersion(latestVersionUrl) - suite.assert.Empty(out) + latestVersionUrl := strings.Replace(common.GitHubLatestReleaseURL, "Azure/azure-storage-fuse", "Azure/azure-storage-fuse-invalid", 1) + out, err := getGitHubLatestRemoteVersion(latestVersionUrl) + suite.assert.Nil(out) suite.assert.Error(err) - suite.assert.Contains(err.Error(), "unable to get latest version") + suite.assert.Contains(err.Error(), "error in GitHub GET latest release") } func getDummyVersion() string { return "1.0.0" } -func (suite *rootCmdSuite) TestGetRemoteVersionValidContainer() { +func (suite *rootCmdSuite) TestGetRemoteVersionValidRepo() { defer suite.cleanupTest() - latestVersionUrl := common.Blobfuse2ListContainerURL + "/latest/index.xml" - out, err := getRemoteVersion(latestVersionUrl) + latestVersionUrl := common.GitHubLatestReleaseURL + out, err := getGitHubLatestRemoteVersion(latestVersionUrl) suite.assert.NotEmpty(out) suite.assert.NoError(err) } diff --git a/common/version.go b/common/version.go index 5787fac8d..16ae4f846 100644 --- a/common/version.go +++ b/common/version.go @@ -39,9 +39,18 @@ import ( "strings" ) -const Blobfuse2ListContainerURL = "https://blobfuse2.z13.web.core.windows.net/release" -const BlobFuse2WarningsURL = "https://aka.ms/blobfuse2warnings" -const BlobFuse2BlockingURL = "https://aka.ms/blobfuse2blockers" +const ( + BlobFuse2WarningsURL = "https://aka.ms/blobfuse2warnings" + BlobFuse2BlockingURL = "https://aka.ms/blobfuse2blockers" + + githubAPI = "https://api.github.com/repos/Azure/azure-storage-fuse" + + // GitHub REST API endpoint for latest release + GitHubLatestReleaseURL = githubAPI + "/releases/latest" + + // GitHub REST API endpoint for release directory contents (security warnings, blocked versions) + GitHubReleaseContentsURL = githubAPI + "/contents/release" +) type Version struct { segments []int64 @@ -134,3 +143,8 @@ func (v Version) OlderThan(v2 Version) bool { func (v Version) NewerThan(v2 Version) bool { return v.compare(v2) == 1 } + +// return the original version string +func (v Version) String() string { + return v.original +} diff --git a/component/azstorage/utils_test.go b/component/azstorage/utils_test.go index 9123b71b0..8db795b5b 100644 --- a/component/azstorage/utils_test.go +++ b/component/azstorage/utils_test.go @@ -277,7 +277,6 @@ func (s *utilsTestSuite) TestBlockNonProxyOptions() { opt, err := getAzBlobServiceClientOptions(&AzStorageConfig{}) assert.NoError(err) assert.EqualValues(0, opt.Retry.MaxRetries) - assert.GreaterOrEqual(len(opt.Logging.AllowedHeaders), 1) } func (s *utilsTestSuite) TestBlockProxyOptions() { @@ -285,17 +284,14 @@ func (s *utilsTestSuite) TestBlockProxyOptions() { opt, err := getAzBlobServiceClientOptions(&AzStorageConfig{proxyAddress: "127.0.0.1", maxRetries: 3}) assert.NoError(err) assert.EqualValues(3, opt.Retry.MaxRetries) - assert.GreaterOrEqual(len(opt.Logging.AllowedHeaders), 1) opt, err = getAzBlobServiceClientOptions(&AzStorageConfig{proxyAddress: "http://127.0.0.1:8080", maxRetries: 3}) assert.NoError(err) assert.EqualValues(3, opt.Retry.MaxRetries) - assert.GreaterOrEqual(len(opt.Logging.AllowedHeaders), 1) opt, err = getAzBlobServiceClientOptions(&AzStorageConfig{proxyAddress: "https://128.0.0.1:8080", maxRetries: 3}) assert.NoError(err) assert.EqualValues(3, opt.Retry.MaxRetries) - assert.GreaterOrEqual(len(opt.Logging.AllowedHeaders), 1) } func (s *utilsTestSuite) TestBfsNonProxyOptions() { @@ -303,7 +299,6 @@ func (s *utilsTestSuite) TestBfsNonProxyOptions() { opt, err := getAzDatalakeServiceClientOptions(&AzStorageConfig{}) assert.NoError(err) assert.EqualValues(0, opt.Retry.MaxRetries) - assert.GreaterOrEqual(len(opt.Logging.AllowedHeaders), 1) } func (s *utilsTestSuite) TestBfsProxyOptions() { @@ -311,17 +306,14 @@ func (s *utilsTestSuite) TestBfsProxyOptions() { opt, err := getAzDatalakeServiceClientOptions(&AzStorageConfig{proxyAddress: "127.0.0.1", maxRetries: 3}) assert.NoError(err) assert.EqualValues(3, opt.Retry.MaxRetries) - assert.GreaterOrEqual(len(opt.Logging.AllowedHeaders), 1) opt, err = getAzDatalakeServiceClientOptions(&AzStorageConfig{proxyAddress: "http://127.0.0.1:8080", maxRetries: 3}) assert.NoError(err) assert.EqualValues(3, opt.Retry.MaxRetries) - assert.GreaterOrEqual(len(opt.Logging.AllowedHeaders), 1) opt, err = getAzDatalakeServiceClientOptions(&AzStorageConfig{proxyAddress: "https://128.0.0.1:8080", maxRetries: 3}) assert.NoError(err) assert.EqualValues(3, opt.Retry.MaxRetries) - assert.GreaterOrEqual(len(opt.Logging.AllowedHeaders), 1) } type endpointAccountType struct { From 9724693eaa97ec0b7e9ebcdc788f1e86f5b87bcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:42:20 +0530 Subject: [PATCH 24/95] Bump azure/login from 2 to 3 (#2164) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/mirror-to-blob.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mirror-to-blob.yml b/.github/workflows/mirror-to-blob.yml index 61fcf9a62..059dedf25 100644 --- a/.github/workflows/mirror-to-blob.yml +++ b/.github/workflows/mirror-to-blob.yml @@ -24,7 +24,7 @@ jobs: # ✅ Azure authentication (OIDC) - name: Azure login - uses: azure/login@v2 + uses: azure/login@v3 with: client-id: ${{ secrets.AZURE_MCP_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_MCP_TENANT_ID }} From 15074e2f62bf921087c4c6c08d58ce05521f82d3 Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:18:53 +0530 Subject: [PATCH 25/95] Update logic for version check (#2163) --- .github/workflows/update-latest-version.yml | 75 +++++++++++ cmd/root.go | 133 +++++++------------- cmd/root_test.go | 91 +++++++------- common/version.go | 17 +-- release/blockedversions/1.1.1 | 0 release/blockedversions/2.3.0 | 0 release/securitywarnings/1.1.1 | 0 release/securitywarnings/2.2.0 | 0 release/securitywarnings/2.2.1 | 0 release/securitywarnings/2.3.0 | 0 release/securitywarnings/2.3.0~preview.1 | 0 release/securitywarnings/2.3.1~preview.1 | 0 12 files changed, 177 insertions(+), 139 deletions(-) create mode 100644 .github/workflows/update-latest-version.yml delete mode 100644 release/blockedversions/1.1.1 delete mode 100644 release/blockedversions/2.3.0 delete mode 100644 release/securitywarnings/1.1.1 delete mode 100644 release/securitywarnings/2.2.0 delete mode 100644 release/securitywarnings/2.2.1 delete mode 100644 release/securitywarnings/2.3.0 delete mode 100644 release/securitywarnings/2.3.0~preview.1 delete mode 100644 release/securitywarnings/2.3.1~preview.1 diff --git a/.github/workflows/update-latest-version.yml b/.github/workflows/update-latest-version.yml new file mode 100644 index 000000000..9745c2293 --- /dev/null +++ b/.github/workflows/update-latest-version.yml @@ -0,0 +1,75 @@ +# This workflow updates the latest-version sentinel file on the benchmarks +# branch whenever a new GA (non-preview) release tag is created. +# +# The blobfuse2 binary checks raw.githubusercontent.com for the file +# release/latest/ +# on the benchmarks branch to determine whether the running version is +# outdated. This workflow keeps that file in sync with the newest release. +# +# Trigger: creation of a tag matching "blobfuse2-*" that does NOT contain +# "preview" (e.g. blobfuse2-2.5.3). Preview tags are intentionally excluded. + +name: Update Latest Version + +on: + push: + tags: + - 'blobfuse2-*' + +jobs: + update-latest-version: + # Skip preview / pre-release tags. + if: "!contains(github.ref_name, 'preview')" + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + # Checkout the benchmarks branch where release metadata lives. + - name: Checkout benchmarks branch + uses: actions/checkout@v4 + with: + ref: benchmarks + fetch-depth: 0 + + # Extract version from the tag (strip "blobfuse2-" prefix), remove + # the old sentinel file, and create the new one. + - name: Update latest version file + run: | + TAG_NAME="${GITHUB_REF_NAME}" + NEW_VERSION="${TAG_NAME#blobfuse2-}" + echo "New release version: ${NEW_VERSION}" + + cd release/latest/ + echo "Before cleanup:" + ls -l + + # Remove old latest-version sentinel file(s) + rm -f * + + echo "After cleanup:" + ls -l + + # Create new empty sentinel file named after the released version + touch "${NEW_VERSION}" + + echo "After update:" + ls -l + + # Commit and push the change to the benchmarks branch. + - name: Commit and push to benchmarks + run: | + TAG_NAME="${GITHUB_REF_NAME}" + NEW_VERSION="${TAG_NAME#blobfuse2-}" + + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + git add release/latest/ + + # Only commit and push if there are staged changes. + if git diff --cached --quiet; then + echo "No changes to commit; skipping commit and push." + else + git commit -m "Update latest version to ${NEW_VERSION}" + git push origin benchmarks + fi diff --git a/cmd/root.go b/cmd/root.go index aee6d72b2..3b3be1dd9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,8 +34,6 @@ package cmd import ( - "encoding/json" - "encoding/xml" "errors" "fmt" "net/http" @@ -50,15 +48,6 @@ import ( "github.com/spf13/cobra" ) -type VersionFilesList struct { - Version string `xml:"latest"` -} - -type Blob struct { - XMLName xml.Name `xml:"Blob"` - Name string `xml:"Name"` -} - var disableVersionCheck bool var rootCmd = &cobra.Command{ @@ -78,138 +67,108 @@ var rootCmd = &cobra.Command{ }, } -// check if the version file exists in the GitHub release directory -func checkVersionExists(versionUrl string) bool { +// checkVersionExists checks whether a file exists at the given raw GitHub URL +// by issuing an HTTP HEAD request. This is used to probe files under: +// - release/latest/ – to determine if the running version is the latest +// - release/securitywarnings/ – to determine if the version has known security issues +// - release/blockedversions/ – to determine if the version is blocked from use +// +// HEAD is preferred over GET because we only need the HTTP status code, not the +// file body. raw.githubusercontent.com returns 200 when the file exists and 404 +// when it does not; no authentication is required for public repos. +func checkVersionExists(fileUrl string) bool { client := &http.Client{ Timeout: 30 * time.Second, Transport: getTransport(), } - req, err := http.NewRequest("GET", versionUrl, nil) + req, err := http.NewRequest("HEAD", fileUrl, nil) if err != nil { log.Err("checkVersionExists: error creating request [%s]", err.Error()) return false } - req.Header.Set("Accept", "application/vnd.github+json") - req.Header.Set("X-GitHub-Api-Version", "2022-11-28") - resp, err := client.Do(req) if err != nil { - log.Err("checkVersionExists: error getting version file from GitHub [%s]", err.Error()) + log.Err("checkVersionExists: error checking version file [%s]", err.Error()) return false } defer resp.Body.Close() - // Treat only 2xx as "exists". 404 is a normal "does not exist" case. + // 2xx means the file exists on the benchmarks branch. if resp.StatusCode >= 200 && resp.StatusCode < 300 { return true } + // 404 is the normal "file does not exist" case. if resp.StatusCode == http.StatusNotFound { return false } - // For other status codes (e.g., 401, 403, 5xx), log and treat as unknown/non-existent. - log.Err("checkVersionExists: unexpected status code from GitHub [%d] for URL %s", resp.StatusCode, versionUrl) + // For other status codes (e.g., 403, 5xx) log error and treat as non-existent. + log.Err("checkVersionExists: unexpected status code [%d] for URL %s", resp.StatusCode, fileUrl) return false } -func getGitHubLatestRemoteVersion(apiEndpoint string) (*common.Version, error) { - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: getTransport(), - } - - // Get Request - req, err := http.NewRequest("GET", apiEndpoint, nil) - if err != nil { - return nil, err - } - - req.Header.Set("Accept", "application/vnd.github+json") - req.Header.Set("X-GitHub-Api-Version", "2022-11-28") - req.Header.Set("User-Agent", "blobfuse2-version-check") - - resp, err := client.Do(req) - if err != nil { - log.Err("getGitHubLatestRemoteVersion: error in GitHub GET latest release [%s]", err.Error()) - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("getGitHubLatestRemoteVersion: error in GitHub GET latest release: %s", resp.Status) - } - - var release struct { // JSON response representation - TagName string `json:"tag_name"` - } - - decoder := json.NewDecoder(resp.Body) - err = decoder.Decode(&release) - if err != nil { - return nil, err - } - - // Remove blobfuse2- prefix in TagName, convert str to Version - // Tag name example, blobfuse2-2.5.2 - versionStr := strings.TrimPrefix(release.TagName, "blobfuse2-") - return common.ParseVersion(versionStr) -} - -// beginDetectNewVersion : Get latest release version and compare if user needs an upgrade or not +// beginDetectNewVersion checks whether the current blobfuse2 version is the +// latest, has known security warnings, or has been blocked entirely. +// +// Instead of calling the GitHub REST API (which is subject to aggressive +// rate-limiting / 429 errors), we check for sentinel files served by +// raw.githubusercontent.com from the "benchmarks" branch under release/: +// +// release/latest/ – exists only for the current latest GA version +// release/securitywarnings/ – exists if this version has known issues +// release/blockedversions/ – exists if this version must not be used func beginDetectNewVersion() chan any { completed := make(chan any) stderr := os.Stderr go func() { defer close(completed) - remote, err := getGitHubLatestRemoteVersion(common.GitHubLatestReleaseURL) - if err != nil { - log.Err("beginDetectNewVersion: error getting latest version [%s]", err.Error()) - if strings.Contains(err.Error(), "no such host") { - log.Err("beginDetectNewVersion: check your network connection and proxy settings") - } - completed <- err.Error() - return - } - - local, err := common.ParseVersion(common.Blobfuse2Version) + // Validate that the compiled-in version string is well-formed. + _, err := common.ParseVersion(common.Blobfuse2Version) if err != nil { log.Err("beginDetectNewVersion: error parsing Blobfuse2Version [%s]", err.Error()) completed <- err.Error() return } - warningsUrl := common.GitHubReleaseContentsURL + "/securitywarnings/" + common.Blobfuse2Version + // --- Security warnings check --- + // If a file release/securitywarnings/ exists, this version + // has known issues that the user should be aware of. + warningsUrl := common.GitHubReleaseBaseURL + "/securitywarnings/" + common.Blobfuse2Version hasWarnings := checkVersionExists(warningsUrl) if hasWarnings { - // This version has known issues associated with it - // Check whether the version has been blocked by the dev team or not. - blockedVersions := common.GitHubReleaseContentsURL + "/blockedversions/" + common.Blobfuse2Version - isBlocked := checkVersionExists(blockedVersions) + // This version has known issues associated with it. + // Check whether the version has been blocked by the dev team. + blockedUrl := common.GitHubReleaseBaseURL + "/blockedversions/" + common.Blobfuse2Version + isBlocked := checkVersionExists(blockedUrl) if isBlocked { - // This version is blocked and customer shall not be allowed to use this. + // This version is blocked and customer shall not be allowed to use it. blockedPage := common.BlobFuse2BlockingURL + "#" + strings.ReplaceAll(strings.ReplaceAll(common.Blobfuse2Version, ".", ""), "~", "") fmt.Fprintf(stderr, "PANIC: Visit %s to see the list of known issues blocking your current version [%s]\n", blockedPage, common.Blobfuse2Version) log.Warn("PANIC: Visit %s to see the list of known issues blocking your current version [%s]\n", blockedPage, common.Blobfuse2Version) os.Exit(1) } else { - // This version is not blocked but has know issues list which customer shall visit. + // This version is not blocked but has a known-issues list which the customer should visit. warningsPage := common.BlobFuse2WarningsURL + "#" + strings.ReplaceAll(strings.ReplaceAll(common.Blobfuse2Version, ".", ""), "~", "") fmt.Fprintf(stderr, "WARNING: Visit %s to see the list of known issues associated with your current version [%s]\n", warningsPage, common.Blobfuse2Version) log.Warn("WARNING: Visit %s to see the list of known issues associated with your current version [%s]\n", warningsPage, common.Blobfuse2Version) } } - if local.OlderThan(*remote) { + // --- Latest-version check --- + // If release/latest/ does NOT exist the running + // version is outdated and a newer release is available. + latestUrl := common.GitHubReleaseBaseURL + "/latest/" + common.Blobfuse2Version + isLatest := checkVersionExists(latestUrl) + if !isLatest { executablePathSegments := strings.Split(strings.ReplaceAll(os.Args[0], "\\", "/"), "/") executableName := executablePathSegments[len(executablePathSegments)-1] - log.Info("beginDetectNewVersion: A new version of Blobfuse2 is available. Current Version=%s, Latest Version=%s", common.Blobfuse2Version, remote.String()) - fmt.Fprintf(stderr, "*** "+executableName+": A new version [%s] is available. Consider upgrading to latest version for bug-fixes & new features. ***\n", remote.String()) - log.Info("*** "+executableName+": A new version [%s] is available. Consider upgrading to latest version for bug-fixes & new features. ***\n", remote.String()) + log.Info("beginDetectNewVersion: A new version of Blobfuse2 is available. Current Version=%s", common.Blobfuse2Version) + fmt.Fprintf(stderr, "*** %s: A new version is available. Current version [%s] is outdated. Consider upgrading to the latest version for bug-fixes & new features. ***\n", executableName, common.Blobfuse2Version) completed <- "A new version of Blobfuse2 is available" } diff --git a/cmd/root_test.go b/cmd/root_test.go index c0ead9085..22378c29f 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -60,7 +60,6 @@ func (suite *rootCmdSuite) SetupTest() { if err != nil { panic("Unable to set silent logger as default.") } - // suite.testExecute() } func (suite *rootCmdSuite) cleanupTest() { @@ -83,93 +82,95 @@ func (suite *rootCmdSuite) TestNoOptionsNoVersionCheck() { suite.assert.Error(err) } +// TestCheckVersionExistsInvalidURL verifies that a completely invalid URL +// returns false rather than panicking. func (suite *rootCmdSuite) TestCheckVersionExistsInvalidURL() { defer suite.cleanupTest() - found := checkVersionExists("abcd") + found := checkVersionExists("://bad-url") suite.assert.False(found) } +// TestNoSecurityWarnings verifies that the current build version has +// no security-warning file on the benchmarks branch. func (suite *rootCmdSuite) TestNoSecurityWarnings() { defer suite.cleanupTest() - warningsUrl := common.GitHubReleaseContentsURL + "/securitywarnings/" + common.Blobfuse2Version + warningsUrl := common.GitHubReleaseBaseURL + "/securitywarnings/" + common.Blobfuse2Version found := checkVersionExists(warningsUrl) suite.assert.False(found) } +// TestSecurityWarnings verifies that version 1.1.1 has a security-warning +// file on the benchmarks branch (live HTTP call). func (suite *rootCmdSuite) TestSecurityWarnings() { defer suite.cleanupTest() - warningsUrl := common.GitHubReleaseContentsURL + "/securitywarnings/" + "1.1.1" + warningsUrl := common.GitHubReleaseBaseURL + "/securitywarnings/" + "1.1.1" found := checkVersionExists(warningsUrl) suite.assert.True(found) } +// TestBlockedVersion verifies that version 1.1.1 has a blocked-version +// file on the benchmarks branch (live HTTP call). func (suite *rootCmdSuite) TestBlockedVersion() { defer suite.cleanupTest() - warningsUrl := common.GitHubReleaseContentsURL + "/blockedversions/" + "1.1.1" - isBlocked := checkVersionExists(warningsUrl) + blockedUrl := common.GitHubReleaseBaseURL + "/blockedversions/" + "1.1.1" + isBlocked := checkVersionExists(blockedUrl) suite.assert.True(isBlocked) } +// TestNonBlockedVersion verifies that the current build version is NOT in the +// blocked-versions list on the benchmarks branch. func (suite *rootCmdSuite) TestNonBlockedVersion() { defer suite.cleanupTest() - warningsUrl := common.GitHubReleaseContentsURL + "/blockedversions/" + common.Blobfuse2Version - found := checkVersionExists(warningsUrl) + blockedUrl := common.GitHubReleaseBaseURL + "/blockedversions/" + common.Blobfuse2Version + found := checkVersionExists(blockedUrl) suite.assert.False(found) } -func (suite *rootCmdSuite) TestGetRemoteVersionInvalidURL() { - defer suite.cleanupTest() - out, err := getGitHubLatestRemoteVersion("abcd") - suite.assert.Nil(out) - suite.assert.Error(err) -} +// TODO: uncomment this after release +// TestLatestVersionExists verifies that the file release/latest/{Blobfuse2Version} is +// present on the benchmarks branch (it is the current GA latest). +// func (suite *rootCmdSuite) TestLatestVersionExists() { +// defer suite.cleanupTest() +// latestUrl := common.GitHubReleaseBaseURL + "/latest/" + common.Blobfuse2Version +// found := checkVersionExists(latestUrl) +// suite.assert.True(found) +// } -func (suite *rootCmdSuite) TestGetRemoteVersionInvalidRepo() { +// TestLatestVersionNotExists verifies that an unknown/old version does NOT +// have a file under release/latest/. +func (suite *rootCmdSuite) TestLatestVersionNotExists() { defer suite.cleanupTest() - latestVersionUrl := strings.Replace(common.GitHubLatestReleaseURL, "Azure/azure-storage-fuse", "Azure/azure-storage-fuse-invalid", 1) - out, err := getGitHubLatestRemoteVersion(latestVersionUrl) - suite.assert.Nil(out) - suite.assert.Error(err) - suite.assert.Contains(err.Error(), "error in GitHub GET latest release") + latestUrl := common.GitHubReleaseBaseURL + "/latest/1.0.0" + found := checkVersionExists(latestUrl) + suite.assert.False(found) } func getDummyVersion() string { return "1.0.0" } -func (suite *rootCmdSuite) TestGetRemoteVersionValidRepo() { - defer suite.cleanupTest() - latestVersionUrl := common.GitHubLatestReleaseURL - out, err := getGitHubLatestRemoteVersion(latestVersionUrl) - suite.assert.NotEmpty(out) - suite.assert.NoError(err) -} - -func (suite *rootCmdSuite) TestGetRemoteVersionCurrentOlder() { +// TestDetectNewVersionCurrentOlder sets the current version to a dummy old +// value (1.0.0) so that release/latest/1.0.0 returns 404, which triggers the +// "new version available" message. +func (suite *rootCmdSuite) TestDetectNewVersionCurrentOlder() { defer suite.cleanupTest() + savedVersion := common.Blobfuse2Version common.Blobfuse2Version = getDummyVersion() + defer func() { common.Blobfuse2Version = savedVersion }() + msg := <-beginDetectNewVersion() suite.assert.NotEmpty(msg) suite.assert.Contains(msg, "A new version of Blobfuse2 is available") } -func (suite *rootCmdSuite) TestGetRemoteVersionCurrentSame() { - defer suite.cleanupTest() - common.Blobfuse2Version = common.Blobfuse2Version_() - msg := <-beginDetectNewVersion() - suite.assert.Nil(msg) -} - -// func (suite *rootCmdSuite) testExecute() { +// TestDetectNewVersionCurrentLatest sets the current version to the actual +// latest on the benchmarks branch so that release/latest/{Blobfuse2Version} +// exists and no upgrade message is produced. +// func (suite *rootCmdSuite) TestDetectNewVersionCurrentLatest() { // defer suite.cleanupTest() -// buf := new(bytes.Buffer) -// rootCmd.SetOut(buf) -// rootCmd.SetErr(buf) -// rootCmd.SetArgs([]string{"--version"}) - -// err := Execute() -// suite.assert.NoError(err) -// suite.assert.Contains(buf.String(), "blobfuse2 version") +// common.Blobfuse2Version = common.Blobfuse2Version_() +// msg := <-beginDetectNewVersion() +// suite.assert.Nil(msg) // } func (suite *rootCmdSuite) TestParseArgs() { diff --git a/common/version.go b/common/version.go index 16ae4f846..749db7f2d 100644 --- a/common/version.go +++ b/common/version.go @@ -43,13 +43,16 @@ const ( BlobFuse2WarningsURL = "https://aka.ms/blobfuse2warnings" BlobFuse2BlockingURL = "https://aka.ms/blobfuse2blockers" - githubAPI = "https://api.github.com/repos/Azure/azure-storage-fuse" - - // GitHub REST API endpoint for latest release - GitHubLatestReleaseURL = githubAPI + "/releases/latest" - - // GitHub REST API endpoint for release directory contents (security warnings, blocked versions) - GitHubReleaseContentsURL = githubAPI + "/contents/release" + // GitHubReleaseBaseURL is the base URL for raw file access on the benchmarks + // branch. Version metadata files live under release/ in three sub-directories: + // release/latest/ – exists only for the latest GA version + // release/securitywarnings/ – exists if this version has known issues + // release/blockedversions/ – exists if this version must not be used + // + // We use raw.githubusercontent.com instead of the GitHub REST API + // (api.github.com) to avoid authentication requirements and aggressive + // rate-limiting / 429-throttling that affects CI pipelines. + GitHubReleaseBaseURL = "https://raw.githubusercontent.com/Azure/azure-storage-fuse/benchmarks/release" ) type Version struct { diff --git a/release/blockedversions/1.1.1 b/release/blockedversions/1.1.1 deleted file mode 100644 index e69de29bb..000000000 diff --git a/release/blockedversions/2.3.0 b/release/blockedversions/2.3.0 deleted file mode 100644 index e69de29bb..000000000 diff --git a/release/securitywarnings/1.1.1 b/release/securitywarnings/1.1.1 deleted file mode 100644 index e69de29bb..000000000 diff --git a/release/securitywarnings/2.2.0 b/release/securitywarnings/2.2.0 deleted file mode 100644 index e69de29bb..000000000 diff --git a/release/securitywarnings/2.2.1 b/release/securitywarnings/2.2.1 deleted file mode 100644 index e69de29bb..000000000 diff --git a/release/securitywarnings/2.3.0 b/release/securitywarnings/2.3.0 deleted file mode 100644 index e69de29bb..000000000 diff --git a/release/securitywarnings/2.3.0~preview.1 b/release/securitywarnings/2.3.0~preview.1 deleted file mode 100644 index e69de29bb..000000000 diff --git a/release/securitywarnings/2.3.1~preview.1 b/release/securitywarnings/2.3.1~preview.1 deleted file mode 100644 index e69de29bb..000000000 From 7aec08cce8cb7150bce261c0744e51eb7e70b4f7 Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:39:29 +0530 Subject: [PATCH 26/95] Updating codeowners (#2165) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d01a89c24..dee0fd1e8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @vibhansa-msft @souravgupta-msft @ashruti-msft @syeleti-msft @jainakanksha-msft +* @vibhansa-msft @souravgupta-msft @syeleti-msft @jainakanksha-msft From 857ea6a6d98fa695d8848ccaea666715c8e3a929 Mon Sep 17 00:00:00 2001 From: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> Date: Wed, 18 Mar 2026 12:40:33 +0530 Subject: [PATCH 27/95] Adding action to create docker container and publish on github container registry (#2162) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/publish-container.yml | 82 +++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/publish-container.yml diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml new file mode 100644 index 000000000..957d1abf7 --- /dev/null +++ b/.github/workflows/publish-container.yml @@ -0,0 +1,82 @@ +# To pull the latest image from GitHub Container Registry, use the following command: +# docker pull ghcr.io/azure/azure-storage-fuse:latest + +name: Build and Publish Container Image + +on: + push: + tags: + - 'v*' + - 'blobfuse2-*' + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: azure/blobfuse2 + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install FUSE dependencies + run: | + sudo apt-get update + sudo apt-get install -y libfuse3-dev fuse3 + + - name: Build blobfuse2 binary + run: ./build.sh + + - name: Prepare Docker build context + run: | + cp blobfuse2 docker/ + cp setup/11-blobfuse2.conf docker/ + cp setup/blobfuse2-logrotate docker/ + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}},value={{tag}},prefix=v + type=semver,pattern={{major}}.{{minor}},value={{tag}},prefix=v + type=semver,pattern={{version}},value={{tag}},prefix=blobfuse- + type=semver,pattern={{major}}.{{minor}},value={{tag}},prefix=blobfuse- + type=ref,event=tag + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: docker + file: docker/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Clean up build artifacts + run: | + rm -f docker/blobfuse2 + rm -f docker/11-blobfuse2.conf + rm -f docker/blobfuse2-logrotate From 4746b40de2d8d5f8f130963a2eb0683ac20191fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 10:28:56 +0530 Subject: [PATCH 28/95] Bump docker/login-action from 3 to 4 (#2170) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index 957d1abf7..27c535589 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -48,7 +48,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} From 3dd5ff41802afb0ac96fa514116fbbaea0dfe00c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 10:55:14 +0530 Subject: [PATCH 29/95] Bump docker/metadata-action from 5 to 6 (#2166) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> --- .github/workflows/publish-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index 27c535589..fec18f90b 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -56,7 +56,7 @@ jobs: - name: Extract metadata (tags, labels) id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | From 7d6057b054411d6f956a0301bdd8cb8b6869a770 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:13:10 +0530 Subject: [PATCH 30/95] Bump actions/checkout from 4 to 6 (#2168) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> --- .github/workflows/publish-container.yml | 2 +- .github/workflows/update-latest-version.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index fec18f90b..69cdd6d7f 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Go uses: actions/setup-go@v5 diff --git a/.github/workflows/update-latest-version.yml b/.github/workflows/update-latest-version.yml index 9745c2293..7c9ee1a76 100644 --- a/.github/workflows/update-latest-version.yml +++ b/.github/workflows/update-latest-version.yml @@ -27,7 +27,7 @@ jobs: steps: # Checkout the benchmarks branch where release metadata lives. - name: Checkout benchmarks branch - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: ref: benchmarks fetch-depth: 0 From c69ac4e9c4fe15c0442dbca32dae0783d6fe9b66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:35:35 +0530 Subject: [PATCH 31/95] Bump docker/build-push-action from 5 to 7 (#2167) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> --- .github/workflows/publish-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index 69cdd6d7f..743a00dcf 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -67,7 +67,7 @@ jobs: type=ref,event=tag - name: Build and push Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v7 with: context: docker file: docker/Dockerfile From b1e2404c51e1e29968cb7d1e584e328fbc00c987 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:39:59 +0530 Subject: [PATCH 32/95] Bump docker/setup-buildx-action from 3 to 4 (#2169) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> --- .github/workflows/publish-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index 743a00dcf..72084e010 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -45,7 +45,7 @@ jobs: cp setup/blobfuse2-logrotate docker/ - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log in to GitHub Container Registry uses: docker/login-action@v4 From 09087494df4651da4df465aca2e4c96e37904cba Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:45:07 +0530 Subject: [PATCH 33/95] Fix nightly pipeline errors (#2173) --- azure-pipeline-templates/package-install.yml | 5 +---- blobfuse2-nightly.yaml | 15 ++++++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 04aba486f..2847ceba1 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -93,11 +93,8 @@ steps: - script: | sudo zypper update -y sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse python3 - wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm - sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm + sudo zypper install -y fuse3 fuse3-devel fusermount3 --v - wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm displayName: 'Libfuse Setup ${{ parameters.distro_version }} for SUSE' condition: eq(variables['distro'], 'suse') diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 81a0732cd..ca3649cc7 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -579,13 +579,14 @@ stages: Description: 'Red Hat Enterprise Linux 9.0' AgentName: 'blobfuse-rhel9' tags: 'fuse3' - CentOS-7.9: - distro_version: "CentOS-7.9" - distro: centos - poolName: 'blobfuse-centos-pool' - containerName: "test-cnt-cent-7" - Description: "CentOS 7.9" - AgentName: "blobfuse-centos7" + # Commenting as CentOS 7.9 image is not available in the marketplace currently + # CentOS-7.9: + # distro_version: "CentOS-7.9" + # distro: centos + # poolName: 'blobfuse-centos-pool' + # containerName: "test-cnt-cent-7" + # Description: "CentOS 7.9" + # AgentName: "blobfuse-centos7" CentOS-8.5: distro_version: "CentOS-8.5" distro: centos From 85fdebf0460f90077e907ad5807c64d9523bd168 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 10:38:36 +0530 Subject: [PATCH 34/95] Bump actions/setup-go from 5 to 6 (#2175) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index 72084e010..4dac80513 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v6 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version-file: go.mod From f62b1eed035f4eb6528053161cefc53e09fa7b17 Mon Sep 17 00:00:00 2001 From: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:21:45 +0530 Subject: [PATCH 35/95] Blobfuse-Buddy mcp server integration (#2151) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/blobfuse-buddy.yml | 117 ++++++ .github/workflows/issueMetrics.yml | 62 +++ .github/workflows/mirror-to-blob.yml | 29 +- cmd/mount.go | 2 +- cmd/mount_all.go | 2 +- cmd/mountv1.go | 2 +- component/attr_cache/attr_cache.go | 2 +- component/attr_cache/attr_cache_test.go | 2 +- component/azstorage/azstorage.go | 4 +- component/azstorage/block_blob.go | 12 +- component/azstorage/block_blob_test.go | 12 +- component/azstorage/config.go | 2 +- component/azstorage/datalake_test.go | 14 +- component/entry_cache/entry_cache_test.go | 2 +- component/file_cache/file_cache.go | 8 +- component/file_cache/file_cache_test.go | 2 +- component/xload/xload_test.go | 2 +- go.mod | 10 +- go.sum | 12 +- go_installer.sh | 2 +- scripts/askbuddy.py | 19 + scripts/call_agent.py | 331 ++++++++++++++++ scripts/deepwiki_query.py | 71 ++++ scripts/fetch_public_docs.py | 30 ++ scripts/issueMetrics.py | 366 ++++++++++++++++++ tools/health-monitor/internal/stats_export.go | 2 +- tools/health-monitor/main.go | 2 +- 27 files changed, 1067 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/blobfuse-buddy.yml create mode 100644 .github/workflows/issueMetrics.yml create mode 100644 scripts/askbuddy.py create mode 100644 scripts/call_agent.py create mode 100644 scripts/deepwiki_query.py create mode 100644 scripts/fetch_public_docs.py create mode 100644 scripts/issueMetrics.py diff --git a/.github/workflows/blobfuse-buddy.yml b/.github/workflows/blobfuse-buddy.yml new file mode 100644 index 000000000..bd2408f40 --- /dev/null +++ b/.github/workflows/blobfuse-buddy.yml @@ -0,0 +1,117 @@ +name: Blobfuse Buddy – AI Auto Reply + +on: + workflow_dispatch: + inputs: + event_type: + description: "Mock event type for manual testing" + required: true + default: "issue" + type: choice + options: + - issue + - discussion + title: + description: "Mock title" + required: false + default: "Blobfuse config question: direct-io vs disable-kernel-cache" + type: string + body: + description: "Mock body" + required: false + default: "What is the difference between direct-io and disable-kernel-cache in Blobfuse2, and which option is generally preferred?" + type: string + issues: + types: [opened] + discussion: + types: [created] + +permissions: + id-token: write + contents: read + issues: write + discussions: write + +jobs: + buddy-reply: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure Login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_MCP_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_MCP_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_MCP_SUBSCRIPTION_ID }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + pip install openai azure-identity requests mcp httpx python-dotenv + + - name: Create mock event payload (manual test) + if: ${{ github.event_name == 'workflow_dispatch' }} + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + ORIGINAL_EVENT_PATH: ${{ github.event_path }} + REPOSITORY: ${{ github.repository }} + run: | + python - <<'PY' + import json + import os + + event_name = os.environ["GITHUB_EVENT_NAME"] + event_type = "issue" + title = "Blobfuse config question: direct-io vs disable-kernel-cache" + body = "What is the difference between direct-io and disable-kernel-cache in Blobfuse2, and which option is generally preferred?" + repository = os.environ["REPOSITORY"] + + if event_name == "workflow_dispatch": + with open(os.environ["ORIGINAL_EVENT_PATH"], "r", encoding="utf-8") as event_file: + original_event = json.load(event_file) + inputs = original_event.get("inputs", {}) + event_type = inputs.get("event_type", event_type) + title = inputs.get("title", title) + body = inputs.get("body", body) + + if event_type == "issue": + payload = { + "issue": { + "title": title, + "body": body, + "comments_url": f"https://api.github.com/repos/{repository}/issues/1/comments" + } + } + else: + payload = { + "discussion": { + "title": title, + "body": body, + "node_id": "MOCK_NODE_ID", + "comments_url": f"https://api.github.com/repos/{repository}/discussions/1/comments" + } + } + + with open("mock_event.json", "w", encoding="utf-8") as file: + json.dump(payload, file) + PY + + - name: Call Azure AI Foundry Agent + env: + GITHUB_EVENT_PATH: ${{ github.event_name == 'workflow_dispatch' && format('{0}/mock_event.json', github.workspace) || github.event_path }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && 'true' || 'false' }} + MOCK_TITLE: "Blobfuse config question: direct-io vs disable-kernel-cache" + MOCK_BODY: "What is the difference between direct-io and disable-kernel-cache in Blobfuse2, and which option is generally preferred?" + FOUNDRY_BASE_URL: https://blobfuse-buddy-resource.services.ai.azure.com/api/projects/blobfuse-buddy/applications/blobfuse-buddy/protocols/openai + FOUNDRY_API_VERSION: 2025-11-15-preview + DEEPWIKI_REPO: ${{ github.repository }} + run: | + python scripts/call_agent.py diff --git a/.github/workflows/issueMetrics.yml b/.github/workflows/issueMetrics.yml new file mode 100644 index 000000000..92db71e67 --- /dev/null +++ b/.github/workflows/issueMetrics.yml @@ -0,0 +1,62 @@ +name: Monthly issue metrics +on: + workflow_dispatch: + schedule: + - cron: "0 0 1 * *" + +permissions: + contents: write + +jobs: + build: + name: issue metrics + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install PyGithub + + - name: Generate 90-day issue and PR metrics report + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO_NAME: ${{ github.repository }} + run: | + python scripts/issueMetrics.py --days 90 > issueMetric.txt + + - name: Checkout benchmarks branch + uses: actions/checkout@v4 + with: + ref: benchmarks + path: benchmarks-branch + + - name: Publish report to benchmarks branch + shell: bash + run: | + set -e + report_date=$(date -u +%Y-%m-%d) + + cp issueMetric.txt benchmarks-branch/issueMetric.txt + mkdir -p benchmarks-branch/issueMetrics + cp issueMetric.txt benchmarks-branch/issueMetrics/issueMetric-${report_date}.txt + + cd benchmarks-branch + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add issueMetric.txt issueMetrics/ + + if git diff --cached --quiet; then + echo "No report changes to commit." + exit 0 + fi + + git commit -m "Publish issue metrics report for ${report_date}" + git push origin benchmarks \ No newline at end of file diff --git a/.github/workflows/mirror-to-blob.yml b/.github/workflows/mirror-to-blob.yml index 059dedf25..f3811c634 100644 --- a/.github/workflows/mirror-to-blob.yml +++ b/.github/workflows/mirror-to-blob.yml @@ -1,4 +1,4 @@ -name: Step-2 Mirror GitHub Content to Blob Storage +name: Mirror GitHub Content to Blob Storage on: workflow_dispatch: @@ -51,9 +51,25 @@ jobs: az storage blob upload-batch \ --auth-mode login \ --account-name $AZURE_STORAGE_ACCOUNT \ - --container-name repo-docs \ + --destination repo-docs \ --source wiki \ - --destination wiki + --destination-path wiki + + + # ✅ Upload public documentation + - name: Fetch public documentation + run: | + pip install html2text + python scripts/fetch_public_docs.py + + - name: Upload public documentation + run: | + az storage blob upload-batch \ + --auth-mode login \ + --account-name $AZURE_STORAGE_ACCOUNT \ + --container-name repo-docs \ + --source public \ + --destination public # ✅ Export issues / PRs - name: Set up Python @@ -65,6 +81,7 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install requests + python3 -m pip install html2text # ✅ Restore incremental state and existing JSONL exports so the exporter # can append new/updated records to the full historical files rather @@ -94,7 +111,9 @@ jobs: az storage blob upload-batch \ --auth-mode login \ --account-name $AZURE_STORAGE_ACCOUNT \ - --container-name repo-gh \ + --destination repo-gh \ --source out_github_export \ - --destination . \ --overwrite + + + \ No newline at end of file diff --git a/cmd/mount.go b/cmd/mount.go index 4c9b64dc5..0e19a2d4c 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -449,7 +449,7 @@ var mountCmd = &cobra.Command{ if !disableVersionCheck { err := VersionCheck() if err != nil { - log.Err(err.Error()) + log.Err("%s", err.Error()) } } diff --git a/cmd/mount_all.go b/cmd/mount_all.go index 179d01299..cb132d3f6 100644 --- a/cmd/mount_all.go +++ b/cmd/mount_all.go @@ -140,7 +140,7 @@ func processCommand() error { if !disableVersionCheck { err := VersionCheck() if err != nil { - log.Err(err.Error()) + log.Err("%s", err.Error()) } } diff --git a/cmd/mountv1.go b/cmd/mountv1.go index 963c943a9..a34457a7e 100755 --- a/cmd/mountv1.go +++ b/cmd/mountv1.go @@ -152,7 +152,7 @@ var generateConfigCmd = &cobra.Command{ if !disableVersionCheck { err := VersionCheck() if err != nil { - log.Err(err.Error()) + log.Err("%s", err.Error()) } } resetOptions() diff --git a/component/attr_cache/attr_cache.go b/component/attr_cache/attr_cache.go index b3c325c1b..1fce85313 100644 --- a/component/attr_cache/attr_cache.go +++ b/component/attr_cache/attr_cache.go @@ -379,7 +379,7 @@ func (ac *AttrCache) cacheAttributes(pathList []*internal.ObjAttr) { for _, attr := range pathList { if len(ac.cacheMap) > ac.maxFiles { - log.Debug("AttrCache::cacheAttributes : %s skipping adding path to attribute cache because it is full", pathList) + log.Debug("AttrCache::cacheAttributes : skipping adding path to attribute cache because it is full (size: %d, max: %d)", len(ac.cacheMap), ac.maxFiles) break } diff --git a/component/attr_cache/attr_cache_test.go b/component/attr_cache/attr_cache_test.go index b02733806..8d6e71abc 100644 --- a/component/attr_cache/attr_cache_test.go +++ b/component/attr_cache/attr_cache_test.go @@ -269,7 +269,7 @@ func (suite *attrCacheTestSuite) TestCreateDir() { var paths = []string{"a", "a/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) // This is a little janky but required since testify suite does not support running setup or clean up for subtests. suite.cleanupTest() suite.SetupTest() diff --git a/component/azstorage/azstorage.go b/component/azstorage/azstorage.go index f4c6a2b1b..22a48ee48 100644 --- a/component/azstorage/azstorage.go +++ b/component/azstorage/azstorage.go @@ -132,13 +132,13 @@ func (az *AzStorage) OnConfigChange() { err = ParseAndReadDynamicConfig(az, conf, true) if err != nil { - log.Err("AzStorage::OnConfigChange : failed to reparse config", err.Error()) + log.Err("AzStorage::OnConfigChange : failed to reparse config [%s]", err.Error()) return } err = az.storage.UpdateConfig(az.stConfig) if err != nil { - log.Err("AzStorage::OnConfigChange : failed to UpdateConfig", err.Error()) + log.Err("AzStorage::OnConfigChange : failed to UpdateConfig [%s]", err.Error()) return } diff --git a/component/azstorage/block_blob.go b/component/azstorage/block_blob.go index d98af8470..b710d82f8 100644 --- a/component/azstorage/block_blob.go +++ b/component/azstorage/block_blob.go @@ -450,7 +450,7 @@ func (bb *BlockBlob) RenameDirectory(source string, target string) error { srcPath := removePrefixPath(bb.Config.prefixPath, *blobInfo.Name) err = bb.RenameFile(srcPath, strings.Replace(srcPath, source, target, 1), nil) if err != nil { - log.Err("BlockBlob::RenameDirectory : Failed to rename file %s [%s]", srcPath, err.Error) + log.Err("BlockBlob::RenameDirectory : Failed to rename file %s [%s]", srcPath, err.Error()) } } } @@ -630,7 +630,7 @@ func (bb *BlockBlob) List(prefix string, marker *string, count int32) ([]*intern // APIs that may be affected include IsDirEmpty, ReadDir and StreamDir if err != nil { - log.Err("BlockBlob::List : Failed to list the container with the prefix %s", err.Error) + log.Err("BlockBlob::List : Failed to list the container with the prefix %s: %v", listPath, err) return nil, nil, err } @@ -1179,7 +1179,7 @@ func (bb *BlockBlob) GetFileBlockOffsets(name string) (*common.BlockOffsetList, storageBlockList, err := blobClient.GetBlockList(context.Background(), blockblob.BlockListTypeCommitted, nil) if err != nil { - log.Err("BlockBlob::GetFileBlockOffsets : Failed to get block list %s ", name, err.Error()) + log.Err("BlockBlob::GetFileBlockOffsets : Failed to get block list %s [%s]", name, err.Error()) return &common.BlockOffsetList{}, err } @@ -1584,7 +1584,7 @@ func (bb *BlockBlob) Write(options *internal.WriteFileOptions) error { // WriteFromBuffer should be able to handle the case where now the block is too big and gets split into multiple blocks err := bb.WriteFromBuffer(name, options.Metadata, *dataBuffer) if err != nil { - log.Err("BlockBlob::Write : Failed to upload to blob %s ", name, err.Error()) + log.Err("BlockBlob::Write : Failed to upload to blob %s [%s]", name, err.Error()) return err } // case 2: given offset is within the size of the blob - and the blob consists of multiple blocks @@ -1597,7 +1597,7 @@ func (bb *BlockBlob) Write(options *internal.WriteFileOptions) error { if exceedsFileBlocks { newBufferSize, err = bb.createNewBlocks(fileOffsets, offset, length) if err != nil { - log.Err("BlockBlob::Write : Failed to create new blocks for file %s", name, err.Error()) + log.Err("BlockBlob::Write : Failed to create new blocks for file %s [%s]", name, err.Error()) return err } } @@ -1766,7 +1766,7 @@ func (bb *BlockBlob) GetCommittedBlockList(name string) (*internal.CommittedBloc storageBlockList, err := blobClient.GetBlockList(context.Background(), blockblob.BlockListTypeCommitted, nil) if err != nil { - log.Err("BlockBlob::GetFileBlockOffsets : Failed to get block list %s ", name, err.Error()) + log.Err("BlockBlob::GetFileBlockOffsets : Failed to get block list %s [%s]", name, err.Error()) return nil, err } diff --git a/component/azstorage/block_blob_test.go b/component/azstorage/block_blob_test.go index 17f125f23..75edbc0c4 100644 --- a/component/azstorage/block_blob_test.go +++ b/component/azstorage/block_blob_test.go @@ -410,7 +410,7 @@ func (s *blockBlobTestSuite) TestCreateDir() { // Testing dir and dir/ var paths = []string{generateDirectoryName(), generateDirectoryName() + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { err := s.az.CreateDir(internal.CreateDirOptions{Name: path}) @@ -431,7 +431,7 @@ func (s *blockBlobTestSuite) TestDeleteDir() { // Testing dir and dir/ var paths = []string{generateDirectoryName(), generateDirectoryName() + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { err := s.az.CreateDir(internal.CreateDirOptions{Name: path}) s.assert.NoError(err) @@ -553,7 +553,7 @@ func (s *blockBlobTestSuite) TestIsDirEmpty() { // Testing dir and dir/ var paths = []string{name, name + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { empty := s.az.IsDirEmpty(internal.IsDirEmptyOptions{Name: name}) @@ -606,7 +606,7 @@ func (s *blockBlobTestSuite) TestReadDir() { // Testing dir and dir/ var paths = []string{name, name + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: path}) s.assert.NoError(err) @@ -627,7 +627,7 @@ func (s *blockBlobTestSuite) TestReadDirNoVirtualDirectory() { // Testing dir and dir/ var paths = []string{"", "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: path}) s.assert.NoError(err) @@ -671,7 +671,7 @@ func (s *blockBlobTestSuite) TestReadDirRoot() { // Testing dir and dir/ var paths = []string{"", "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { // ReadDir only reads the first level of the hierarchy entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: path}) diff --git a/component/azstorage/config.go b/component/azstorage/config.go index b04a50b13..757df84f7 100644 --- a/component/azstorage/config.go +++ b/component/azstorage/config.go @@ -535,7 +535,7 @@ func ParseAndValidateConfig(az *AzStorage, opt AzStorageOptions) error { log.Crit("ParseAndValidateConfig : account %s, container %s, account-type %s, auth %s, prefix %s, endpoint %s, MD5 %v %v, virtual-directory %v, disable-compression %v, CPK %v", az.stConfig.authConfig.AccountName, az.stConfig.container, az.stConfig.authConfig.AccountType, az.stConfig.authConfig.AuthMode, az.stConfig.prefixPath, az.stConfig.authConfig.Endpoint, az.stConfig.validateMD5, az.stConfig.updateMD5, az.stConfig.virtualDirectory, az.stConfig.disableCompression, az.stConfig.cpkEnabled) - log.Crit("ParseAndValidateConfig : use-HTTP %t, block-size %d, max-concurrency %d, default-tier %s, fail-unsupported-op %t, mount-all-containers %t", az.stConfig.authConfig.UseHTTP, az.stConfig.blockSize, az.stConfig.maxConcurrency, az.stConfig.defaultTier, az.stConfig.ignoreAccessModifiers, az.stConfig.mountAllContainers) + log.Crit("ParseAndValidateConfig : use-HTTP %t, block-size %d, max-concurrency %d, default-tier %v, fail-unsupported-op %t, mount-all-containers %t", az.stConfig.authConfig.UseHTTP, az.stConfig.blockSize, az.stConfig.maxConcurrency, az.stConfig.defaultTier, az.stConfig.ignoreAccessModifiers, az.stConfig.mountAllContainers) log.Crit("ParseAndValidateConfig : Retry Config: retry-count %d, max-timeout %d, backoff-time %d, max-delay %d, preserve-acl: %v", az.stConfig.maxRetries, az.stConfig.maxTimeout, az.stConfig.backoffTime, az.stConfig.maxRetryDelay, az.stConfig.preserveACL) diff --git a/component/azstorage/datalake_test.go b/component/azstorage/datalake_test.go index b85c32cbf..52df2013e 100644 --- a/component/azstorage/datalake_test.go +++ b/component/azstorage/datalake_test.go @@ -268,7 +268,7 @@ func (s *datalakeTestSuite) TestFNSOverHNS() { var paths = []string{generateDirectoryName(), generateDirectoryName() + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { err := s.az.CreateDir(internal.CreateDirOptions{Name: path}) @@ -290,7 +290,7 @@ func (s *datalakeTestSuite) TestCreateDir() { // Testing dir and dir/ var paths = []string{generateDirectoryName(), generateDirectoryName() + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { err := s.az.CreateDir(internal.CreateDirOptions{Name: path}) @@ -320,7 +320,7 @@ func (s *datalakeTestSuite) TestCreateDirWithCPKEnabled() { // Testing dir and dir/ var paths = []string{generateDirectoryName()} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { err := s.az.CreateDir(internal.CreateDirOptions{Name: path}) @@ -345,7 +345,7 @@ func (s *datalakeTestSuite) TestDeleteDir() { // Testing dir and dir/ var paths = []string{generateDirectoryName(), generateDirectoryName() + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { err := s.az.CreateDir(internal.CreateDirOptions{Name: path}) s.assert.NoError(err) @@ -493,7 +493,7 @@ func (s *datalakeTestSuite) TestIsDirEmpty() { // Testing dir and dir/ var paths = []string{name, name + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { empty := s.az.IsDirEmpty(internal.IsDirEmptyOptions{Name: name}) @@ -546,7 +546,7 @@ func (s *datalakeTestSuite) TestReadDir() { // Testing dir and dir/ var paths = []string{name, name + "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: path}) s.assert.NoError(err) @@ -587,7 +587,7 @@ func (s *datalakeTestSuite) TestReadDirRoot() { // Testing dir and dir/ var paths = []string{"", "/"} for _, path := range paths { - log.Debug(path) + log.Debug("%s", path) s.Run(path, func() { // ReadDir only reads the first level of the hierarchy entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: path}) diff --git a/component/entry_cache/entry_cache_test.go b/component/entry_cache/entry_cache_test.go index 005b267e1..baf4850e8 100644 --- a/component/entry_cache/entry_cache_test.go +++ b/component/entry_cache/entry_cache_test.go @@ -96,7 +96,7 @@ func (suite *entryCacheTestSuite) SetupTest() { rand := randomString(8) suite.fake_storage_path = filepath.Join(home_dir, "fake_storage"+rand) defaultConfig := fmt.Sprintf("read-only: true\n\nentry_cache:\n timeout-sec: 7\n\nloopbackfs:\n path: %s", suite.fake_storage_path) - log.Debug(defaultConfig) + log.Debug("%s", defaultConfig) // Delete the temp directories created os.RemoveAll(suite.fake_storage_path) diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index 05d67a33d..0be38cd10 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -458,7 +458,7 @@ func (fc *FileCache) invalidateDirectory(name string) { // TODO : wouldn't this cause a race condition? a thread might get the lock before we purge - and the file would be non-existent err = filepath.WalkDir(localPath, func(path string, d fs.DirEntry, err error) error { if err == nil && d != nil { - log.Debug("FileCache::invalidateDirectory : %s (%d) getting removed from cache", path, d.IsDir()) + log.Debug("FileCache::invalidateDirectory : %s (%v) getting removed from cache", path, d.IsDir()) if !d.IsDir() { fc.policy.CachePurge(path) } else { @@ -1485,7 +1485,7 @@ func (fc *FileCache) RenameFile(options internal.RenameFileOptions) error { // stale content). We either need to remove dest file as well from cache or just run rename to replace the content. err = os.Rename(localSrcPath, localDstPath) if err != nil && !os.IsNotExist(err) { - log.Err("FileCache::RenameFile : %s failed to rename local file %s [%s]", localSrcPath, err.Error()) + log.Err("FileCache::RenameFile : %s failed to rename local file %s [%s]", localSrcPath, localDstPath, err.Error()) } if err != nil { @@ -1494,7 +1494,7 @@ func (fc *FileCache) RenameFile(options internal.RenameFileOptions) error { // so deleting local dest file ensures next open of that will get the updated file from container err = deleteFile(localDstPath) if err != nil && !os.IsNotExist(err) { - log.Err("FileCache::RenameFile : %s failed to delete local file %s [%s]", localDstPath, err.Error()) + log.Err("FileCache::RenameFile : failed to delete local file %s [%s]", localDstPath, err.Error()) } fc.policy.CachePurge(localDstPath) @@ -1502,7 +1502,7 @@ func (fc *FileCache) RenameFile(options internal.RenameFileOptions) error { err = deleteFile(localSrcPath) if err != nil && !os.IsNotExist(err) { - log.Err("FileCache::RenameFile : %s failed to delete local file %s [%s]", localSrcPath, err.Error()) + log.Err("FileCache::RenameFile : failed to delete local file %s [%s]", localSrcPath, err.Error()) } fc.policy.CachePurge(localSrcPath) diff --git a/component/file_cache/file_cache_test.go b/component/file_cache/file_cache_test.go index c927023ce..5019b242f 100644 --- a/component/file_cache/file_cache_test.go +++ b/component/file_cache/file_cache_test.go @@ -110,7 +110,7 @@ func (suite *fileCacheTestSuite) SetupTest() { suite.cache_path = filepath.Join(home_dir, "file_cache"+rand) suite.fake_storage_path = filepath.Join(home_dir, "fake_storage"+rand) defaultConfig := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n timeout-sec: 0\n\nloopbackfs:\n path: %s", suite.cache_path, suite.fake_storage_path) - log.Debug(defaultConfig) + log.Debug("%s", defaultConfig) // Delete the temp directories created os.RemoveAll(suite.cache_path) diff --git a/component/xload/xload_test.go b/component/xload/xload_test.go index 32f0162c4..219c469c4 100644 --- a/component/xload/xload_test.go +++ b/component/xload/xload_test.go @@ -89,7 +89,7 @@ func (suite *xloadTestSuite) SetupTest() { suite.local_path = filepath.Join("/tmp/", "xload_"+rand) suite.fake_storage_path = filepath.Join("/tmp/", "fake_storage_"+rand) defaultConfig := fmt.Sprintf("xload:\n path: %s\n\nloopbackfs:\n path: %s\n\nread-only: true", suite.local_path, suite.fake_storage_path) - log.Debug(defaultConfig) + log.Debug("%s", defaultConfig) // Delete the temp directories created os.RemoveAll(suite.local_path) diff --git a/go.mod b/go.mod index 9dba18a50..9aed69c6a 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/Azure/azure-storage-fuse/v2 -go 1.25.0 - -toolchain go1.25.7 +go 1.26.1 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 @@ -32,7 +30,7 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect @@ -41,7 +39,7 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 + github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -51,7 +49,7 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/net v0.50.0 // indirect + golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.42.0 golang.org/x/text v0.34.0 // indirect ) diff --git a/go.sum b/go.sum index 587b12f4b..4b8468128 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 h1:7QtoGxKm6mPhs github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4/go.mod h1:juYrzH1q6A+g9ZZbGh0OmjS7zaMq3rFDrPhVnYSgFMA= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 h1:4iB+IesclUXdP0ICgAabvq2FYLXrJWKx1fJQ+GxSo3Y= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNPJ3W56nxOKb4164LEw094U0C8zYQM8mQU= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda/go.mod h1:2CaSFTh2ph9ymS6goiOKIBdfhwWUVsX4nQ5QjIYFHHs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -57,8 +57,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 h1:KPpdlQLZcHfTMQRi6bFQ7ogNO0ltFT4PmtwTLW4W+14= -github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 h1:rh2lKw/P/EqHa724vYH2+VVQ1YnW4u6EOXl0PMAovZE= +github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -113,8 +113,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/go_installer.sh b/go_installer.sh index 3dae86720..7e128a5d8 100755 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,6 +1,6 @@ #!/bin/bash work_dir=$(echo $1 | sed 's:/*$::') -version="1.25.7" +version="1.26.1" arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` if [ $arch != "arm64" ] diff --git a/scripts/askbuddy.py b/scripts/askbuddy.py new file mode 100644 index 000000000..932f91c55 --- /dev/null +++ b/scripts/askbuddy.py @@ -0,0 +1,19 @@ +from openai import OpenAI +from azure.identity import DefaultAzureCredential, get_bearer_token_provider + +token_provider = get_bearer_token_provider( + DefaultAzureCredential(), + "https://ai.azure.com/.default" +) + +client = OpenAI( + api_key=token_provider, + base_url="https://blobfuse-buddy-resource.services.ai.azure.com/api/projects/blobfuse-buddy/applications/blobfuse-buddy/protocols/openai", + default_query={"api-version": "2025-11-15-preview"} +) + +response = client.responses.create( + input="Test request: Should I use direct-io or disable-kernel-cache?" +) + +print(response.output_text) \ No newline at end of file diff --git a/scripts/call_agent.py b/scripts/call_agent.py new file mode 100644 index 000000000..7e415309d --- /dev/null +++ b/scripts/call_agent.py @@ -0,0 +1,331 @@ +import json +import os +import re +import sys +import requests +from pathlib import Path +from openai import OpenAI +from azure.identity import DefaultAzureCredential, get_bearer_token_provider + +# ----------------------------- +# Load GitHub event payload +# ----------------------------- +with open(os.environ["GITHUB_EVENT_PATH"], "r") as f: + event = json.load(f) + +is_issue = "issue" in event +is_discussion = "discussion" in event + +if is_issue: + title = event["issue"]["title"] + body = event["issue"].get("body", "") + comments_url = event["issue"]["comments_url"] +elif is_discussion: + title = event["discussion"]["title"] + body = event["discussion"].get("body", "") + discussion_node_id = event["discussion"]["node_id"] + comments_url = None # discussions use GraphQL, not REST +else: + dry_run = os.getenv("DRY_RUN", "false").lower() == "true" + if not dry_run: + raise RuntimeError("Unsupported GitHub event") + title = os.getenv("MOCK_TITLE", "Dry-run workflow test") + body = os.getenv("MOCK_BODY", "No issue/discussion payload found; using dry-run fallback.") + repository = os.getenv("GITHUB_REPOSITORY", "owner/repo") + comments_url = f"https://api.github.com/repos/{repository}/issues/1/comments" + + +# ----------------------------- +# Query DeepWiki for context (via MCP SSE) +# ----------------------------- +DEEPWIKI_REPO = os.getenv("DEEPWIKI_REPO") # e.g. "Azure/azure-storage-fuse" + +def query_deepwiki(question, repo_name): + """Ask DeepWiki a question about the repo via MCP SSE and return its answer.""" + if not repo_name: + print("DEEPWIKI_REPO not set, skipping DeepWiki query.") + return "" + try: + import subprocess + result = subprocess.run( + [sys.executable, os.path.join(os.path.dirname(__file__), "deepwiki_query.py"), + repo_name, question, ""], + capture_output=True, text=True, timeout=120, + ) + if result.returncode != 0: + print(f"DeepWiki query failed (non-fatal): {result.stderr.strip()}") + return "" + answer = result.stdout.strip() + print(f"DeepWiki returned {len(answer)} chars.") + print("----- DeepWiki Answer -----") + print(answer) + print("----- End DeepWiki Answer -----") + return answer + except Exception as e: + print(f"DeepWiki query failed (non-fatal): {e}") + return "" + + +# ----------------------------- +# Search local repo docs for context +# ----------------------------- +DOC_FILES = [ + "TSG.md", + "README.md", + "MIGRATION.md", + "KnownLimitations.txt", + "CHANGELOG.md", + "setup/baseConfig.yaml", + "setup/advancedConfig.yaml", + "setup/readme.md", + "sampleFileCacheConfig.yaml", + "sampleBlockCacheConfig.yaml", + "sampleFileCacheWithSASConfig.yaml", +] +# Include all CLI docs from doc/ +DOC_DIR = Path("doc") +if DOC_DIR.is_dir(): + DOC_FILES.extend(str(p) for p in DOC_DIR.glob("*.md")) + +MAX_CONTEXT_CHARS = 12000 # cap to avoid overwhelming the prompt + + +def search_local_docs(question): + """Search repo documentation for paragraphs relevant to the question.""" + # Extract keywords (3+ char words, lowercased, deduplicated) + words = set(re.findall(r"[a-z][a-z0-9_-]{2,}", question.lower())) + # Add common blobfuse terms that might appear as sub-strings + extra = {"mount", "unmount", "cache", "fuse", "config", "auth", "sas", + "spn", "msi", "key", "permission", "error", "fail", "timeout"} + keywords = words | (extra & set(question.lower().split())) + + scored_chunks = [] + for filepath in DOC_FILES: + path = Path(filepath) + if not path.is_file(): + continue + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + + # Split into paragraphs (double-newline or markdown heading boundary) + paragraphs = re.split(r"\n{2,}|\n(?=#)", text) + for para in paragraphs: + para = para.strip() + if len(para) < 30: + continue + lower_para = para.lower() + score = sum(1 for kw in keywords if kw in lower_para) + if score > 0: + scored_chunks.append((score, filepath, para)) + + # Sort by relevance and assemble context within budget + scored_chunks.sort(key=lambda x: x[0], reverse=True) + context_parts = [] + total = 0 + for score, source, chunk in scored_chunks: + if total + len(chunk) > MAX_CONTEXT_CHARS: + break + context_parts.append(f"[Source: {source}]\n{chunk}") + total += len(chunk) + + result = "\n\n---\n\n".join(context_parts) + print(f"Local doc search: {len(context_parts)} relevant snippets, {total} chars.") + return result + + +# ----------------------------- +# Gather context +# ----------------------------- +user_question = f"{title}\n{body}" +deepwiki_context = query_deepwiki(user_question, DEEPWIKI_REPO) +local_doc_context = search_local_docs(user_question) + +# ----------------------------- +# Build prompt +# ----------------------------- +context_block = "" +if deepwiki_context: + context_block += f""" +--- DeepWiki Reference Answer --- +{deepwiki_context} +--- End DeepWiki Reference --- +""" +if local_doc_context: + context_block += f""" +--- Relevant Documentation Snippets --- +{local_doc_context} +--- End Documentation --- +""" + +prompt = f""" +New GitHub item received. + +Title: +{title} + +Description: +{body} +{context_block} +Using the reference information above as grounding context, please analyze the issue/question +and provide a helpful, accurate response. Prioritize factual accuracy from the documentation. +Where the documentation doesn't cover the topic, use your own knowledge but note it clearly. +""" + +# ----------------------------- +# Call Azure AI Foundry Agent +# ----------------------------- +dry_run = os.getenv("DRY_RUN", "false").lower() == "true" +token_provider = get_bearer_token_provider( + DefaultAzureCredential(), + "https://ai.azure.com/.default" +) + +client = OpenAI( + api_key=token_provider, + base_url=os.environ["FOUNDRY_BASE_URL"], + default_query={"api-version": os.environ["FOUNDRY_API_VERSION"]} +) + +response = client.responses.create( + input=prompt, + timeout=60 +) + +agent_reply = response.output_text.strip() + +# ----------------------------- +# Strip unwanted sections from agent reply +# ----------------------------- +# Remove "Summary Table" and "Primary Sources" sections (and variants) +agent_reply = re.sub( + r"(?:^|\n)#+\s*\*?\*?Summary\s+Table\*?\*?.*?(?=\n#+\s|\Z)", + "", agent_reply, flags=re.DOTALL | re.IGNORECASE +) +agent_reply = re.sub( + r"(?:^|\n)\*?\*?Summary\s+Table\*?\*?\s*\n\|.*?(?=\n#+\s|\n[^|\s]|\Z)", + "", agent_reply, flags=re.DOTALL | re.IGNORECASE +) +agent_reply = re.sub( + r"(?:^|\n)#+\s*\*?\*?(?:Primary\s+Sources?|References?)\*?\*?.*?(?=\n#+\s|\Z)", + "", agent_reply, flags=re.DOTALL | re.IGNORECASE +) +agent_reply = re.sub( + r"(?:^|\n)\*?\*?(?:Primary\s+Sources?|References?)[:\*]*\*?\*?\s*\n.*?(?=\n#+\s|\n\n[^-*\s]|\Z)", + "", agent_reply, flags=re.DOTALL | re.IGNORECASE +) +agent_reply = agent_reply.strip() + +# ----------------------------- +# Prepend AI disclaimer +# ----------------------------- +final_reply = f"""⚠️ **AI‑generated response** + +This response was generated by an AI agent and may not be fully accurate. +Please validate recommendations before acting on them. + +--- + +{agent_reply} +""" + +# ----------------------------- +# Auto-summarize if too long +# ----------------------------- +GITHUB_COMMENT_CHAR_LIMIT = 65536 +SAFE_LIMIT = 60000 # leave headroom for markdown overhead +SUMMARIZE_TARGET = 55000 # target length for the summarized version + +if len(final_reply) > SAFE_LIMIT: + print(f"Response length ({len(final_reply)} chars) exceeds safe limit ({SAFE_LIMIT}). Requesting summarization...") + summarize_prompt = ( + f"The following answer is too long to post as a single GitHub comment " + f"(limit ~{GITHUB_COMMENT_CHAR_LIMIT} characters). " + f"Please condense it to at most {SUMMARIZE_TARGET} characters while preserving " + f"all key information, code examples, and actionable recommendations. " + f"Keep the same markdown formatting.\n\n" + f"--- ORIGINAL ANSWER ---\n{agent_reply}" + ) + try: + summary_resp = client.responses.create( + input=summarize_prompt, + timeout=90, + ) + summarized = summary_resp.output_text.strip() + final_reply = f"""⚠️ **AI‑generated response** *(summarized — original exceeded comment size limit)* + +This response was generated by an AI agent and may not be fully accurate. +Please validate recommendations before acting on them. + +--- + +{summarized} +""" + print(f"Summarized response to {len(final_reply)} chars.") + except Exception as e: + print(f"Summarization failed ({e}). Falling back to hard truncation.") + +# Hard-truncate as a last-resort safety net +if len(final_reply) > GITHUB_COMMENT_CHAR_LIMIT: + truncation_notice = "\n\n---\n⚠️ *Response was truncated because it exceeded the GitHub comment size limit.*" + max_body = GITHUB_COMMENT_CHAR_LIMIT - len(truncation_notice) + final_reply = final_reply[:max_body] + truncation_notice + print(f"Hard-truncated response to {len(final_reply)} chars.") + +if dry_run: + question = f"Title: {title}\nDescription: {body}" + print("DRY_RUN enabled. Skipping GitHub comment post.") + print(f"Target URL: {comments_url}") + print("----- Validation Q&A -----") + print("Question Asked:") + print(question) + print() + print("Agent Answer:") + print(final_reply) + print("----- End Validation Q&A -----") + raise SystemExit(0) + +# ----------------------------- +# Post reply back to GitHub +# ----------------------------- +headers = { + "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", + "Accept": "application/vnd.github+json", +} + +try: + if is_discussion: + # GitHub discussions require the GraphQL API for commenting + graphql_url = "https://api.github.com/graphql" + mutation = """ + mutation($discussionId: ID!, $body: String!) { + addDiscussionComment(input: {discussionId: $discussionId, body: $body}) { + comment { id url } + } + } + """ + gql_resp = requests.post( + graphql_url, + headers=headers, + json={"query": mutation, "variables": {"discussionId": discussion_node_id, "body": final_reply}}, + ) + gql_resp.raise_for_status() + gql_data = gql_resp.json() + if "errors" in gql_data: + print(f"GraphQL errors: {gql_data['errors']}", file=sys.stderr) + sys.exit(1) + print(f"Discussion comment posted: {gql_data['data']['addDiscussionComment']['comment']['url']}") + else: + # Issues use the REST API + post_resp = requests.post( + comments_url, + headers=headers, + json={"body": final_reply}, + ) + post_resp.raise_for_status() + print(f"Issue comment posted: {post_resp.json().get('html_url', comments_url)}") +except requests.exceptions.HTTPError as e: + print(f"Failed to post comment: {e}", file=sys.stderr) + print(f"Response body: {e.response.text}", file=sys.stderr) + sys.exit(1) \ No newline at end of file diff --git a/scripts/deepwiki_query.py b/scripts/deepwiki_query.py new file mode 100644 index 000000000..9ab014b69 --- /dev/null +++ b/scripts/deepwiki_query.py @@ -0,0 +1,71 @@ +import asyncio +import os +import sys +from contextlib import AsyncExitStack +from typing import Optional +import subprocess + +import httpx +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client +from dotenv import load_dotenv + +load_dotenv() + +# The URL for the DeepWiki MCP server's Streamable HTTP transport +MCP_URL = "https://mcp.deepwiki.com/mcp" + +class MCPClient: + def __init__(self): + self.session: Optional[ClientSession] = None + self._streams_context = None + self._session_context = None + + async def connect_to_server(self, server_url: str): + self._streams_context = streamablehttp_client(url=server_url) + streams = await self._streams_context.__aenter__() + # streamablehttp_client yields (read, write, get_session_id) + self._session_context = ClientSession(streams[0], streams[1]) + self.session = await self._session_context.__aenter__() + await self.session.initialize() + + async def cleanup(self): + if self._session_context: + await self._session_context.__aexit__(None, None, None) + if self._streams_context: + await self._streams_context.__aexit__(None, None, None) + + async def ask_deepwiki(self, repo: str, question: str) -> str: + if not self.session: + raise RuntimeError("Client not connected.") + + # The MCP SDK handles the JSON-RPC call for you + result = await self.session.call_tool( + "ask_question", {"repoName": repo, "question": question} + ) + + # Join the list of content parts into a single string + return result.content + +async def main(repo, title, body): + client = MCPClient() + try: + await client.connect_to_server(server_url=MCP_URL) + + question = f"{title}\n\n{body}" + response = await client.ask_deepwiki(repo, question) + print (response) + + finally: + await client.cleanup() + +if __name__ == "__main__": + if len(sys.argv) < 4: + print("Usage: python deepwiki_query.py ") + sys.exit(1) + + repo_arg = sys.argv[1] + issue_title = sys.argv[2] + issue_body = sys.argv[3] + + asyncio.run(main(repo_arg, issue_title, issue_body)) diff --git a/scripts/fetch_public_docs.py b/scripts/fetch_public_docs.py new file mode 100644 index 000000000..4e846fb98 --- /dev/null +++ b/scripts/fetch_public_docs.py @@ -0,0 +1,30 @@ +import requests +from pathlib import Path +import html2text + +DOCS = { + "blobfuse2-what-is.md": + "https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-what-is", + "blobfuse2-installation.md": + "https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-install?tabs=Ubuntu", + "blobfuse2-mount.md": + "https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-mount-container", + "blobfuse2-compare.md": + "https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-compare-linux-file-system" +} + +out_dir = Path("public") +out_dir.mkdir(exist_ok=True) + +converter = html2text.HTML2Text() +converter.ignore_links = False + +for name, url in DOCS.items(): + print(f"[public-docs] fetching {url}") + r = requests.get(url, timeout=30) + r.raise_for_status() + + md = converter.handle(r.text) + (out_dir / name).write_text(md, encoding="utf-8") + +print("[public-docs] done") \ No newline at end of file diff --git a/scripts/issueMetrics.py b/scripts/issueMetrics.py new file mode 100644 index 000000000..296088fc2 --- /dev/null +++ b/scripts/issueMetrics.py @@ -0,0 +1,366 @@ +import datetime +import argparse +from collections import Counter +import os +import statistics +import subprocess +import sys + +from github import Auth, Github +from github.GithubException import BadCredentialsException, GithubException + + +DEFAULT_DAYS_LOOKBACK = 90 + + +def get_env_var(*names: str) -> str: + for name in names: + value = os.getenv(name, "").strip() + if value: + return value + return "" + + +def get_gh_cli_token() -> str: + try: + result = subprocess.run( + ["gh", "auth", "token"], + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError: + return "" + except subprocess.CalledProcessError: + result = None + + if result is not None: + token = result.stdout.strip() + if token: + return token + + try: + result = subprocess.run( + ["gh", "auth", "status", "--show-token"], + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError: + return "" + except subprocess.CalledProcessError: + return "" + + token_output = "\n".join([result.stdout, result.stderr]) + for line in token_output.splitlines(): + marker = "Token:" + if marker in line: + token = line.split(marker, 1)[1].strip() + if token: + return token + return "" + + +def get_gh_cli_repo() -> str: + try: + result = subprocess.run( + ["gh", "repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"], + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError: + return "" + except subprocess.CalledProcessError: + return "" + return result.stdout.strip() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate GitHub issue metrics report.") + parser.add_argument( + "--days", + type=int, + default=int(os.getenv("DAYS_LOOKBACK", str(DEFAULT_DAYS_LOOKBACK))), + help=f"Number of days to look back (default: {DEFAULT_DAYS_LOOKBACK}).", + ) + parser.add_argument( + "--only-copilot-tagged", + action="store_true", + help="Show only copilot-tagged entries in issue and PR details tables.", + ) + parser.add_argument( + "--summary", + action="store_true", + help="Show only summary tables (hide issue and PR details).", + ) + return parser.parse_args() + + +def print_table(title: str, headers: list[str], rows: list[list[str]]) -> None: + table_rows = [headers] + rows + widths = [max(len(str(row[index])) for row in table_rows) for index in range(len(headers))] + + def format_row(row: list[str]) -> str: + return "| " + " | ".join(str(value).ljust(widths[index]) for index, value in enumerate(row)) + " |" + + separator = "+-" + "-+-".join("-" * width for width in widths) + "-+" + + print(title) + print(separator) + print(format_row(headers)) + print(separator) + for row in rows: + print(format_row(row)) + print(separator) + + +def format_duration(hours: float) -> str: + if hours >= 24: + return f"{hours / 24:.2f} days" + return f"{hours:.2f} hrs" + + +def format_days(hours: float) -> str: + return f"{hours / 24:.2f}" + + +def truncate_text(text: str, max_length: int = 60) -> str: + if len(text) <= max_length: + return text + return text[: max_length - 3] + "..." + + +def is_copilot_tagged(labels) -> bool: + for label in labels: + if hasattr(label, "name") and "copilot" in label.name.lower(): + return True + return False + + +def get_first_comment_time_display_for_pr(pr) -> str: + first_comment_time = None + + try: + issue_comment = next(iter(pr.get_issue_comments()), None) + if issue_comment is not None and issue_comment.created_at is not None: + first_comment_time = issue_comment.created_at + except GithubException: + return "Unavailable" + + try: + review_comment = next(iter(pr.get_comments()), None) + if review_comment is not None and review_comment.created_at is not None: + if first_comment_time is None or review_comment.created_at < first_comment_time: + first_comment_time = review_comment.created_at + except GithubException: + return "Unavailable" + + if first_comment_time is None: + return "No comments" + + first_comment_hours = (first_comment_time - pr.created_at).total_seconds() / 3600 + return format_duration(first_comment_hours) + + +def main() -> int: + args = parse_args() + + access_token = get_env_var("GITHUB_TOKEN", "GH_TOKEN") or get_gh_cli_token() + repo_name = get_env_var("REPO_NAME", "GITHUB_REPOSITORY") or get_gh_cli_repo() + days_lookback = args.days + + if days_lookback <= 0: + print("Error: --days must be a positive integer.", file=sys.stderr) + return 1 + + if not access_token: + print( + "Error: Missing token. Set GITHUB_TOKEN/GH_TOKEN or run 'gh auth login'.", + file=sys.stderr, + ) + return 1 + + if not repo_name or "/" not in repo_name: + print("Error: Missing repo. Set REPO_NAME or GITHUB_REPOSITORY as 'owner/repo'.", file=sys.stderr) + return 1 + + try: + g = Github(auth=Auth.Token(access_token)) + repo = g.get_repo(repo_name) + except BadCredentialsException: + print("Error: GitHub authentication failed (bad token).", file=sys.stderr) + return 1 + except GithubException as ex: + print(f"Error: Unable to access repository '{repo_name}': {ex}", file=sys.stderr) + return 1 + + run_timestamp = datetime.datetime.now(datetime.timezone.utc) + since = run_timestamp - datetime.timedelta(days=days_lookback) + + print(f"Report Generated On: {run_timestamp:%Y-%m-%d %H:%M:%S} UTC") + print(f"Report Window: Last {days_lookback} days") + print() + + issues = repo.get_issues(state="all", since=since) + + new_count = 0 + open_count = 0 + resolved_count = 0 + issue_copilot_yes_count = 0 + resolution_times = [] + issue_rows = [] + + for issue in issues: + if issue.pull_request: + continue + + new_count += 1 + close_time_display = "Open" + first_comment_display = "No comments" + + if issue.comments and issue.comments > 0: + try: + first_comment = next(iter(issue.get_comments()), None) + if first_comment is not None and first_comment.created_at is not None: + first_comment_hours = (first_comment.created_at - issue.created_at).total_seconds() / 3600 + first_comment_display = format_duration(first_comment_hours) + except GithubException: + first_comment_display = "Unavailable" + + if issue.state == "open": + open_count += 1 + elif issue.state == "closed" and issue.closed_at is not None: + resolved_count += 1 + duration = issue.closed_at - issue.created_at + resolution_hours = duration.total_seconds() / 3600 + resolution_times.append(resolution_hours) + close_time_display = format_days(resolution_hours) + + issue_copilot_tag = "Yes" if is_copilot_tagged(issue.labels) else "No" + if issue_copilot_tag == "Yes": + issue_copilot_yes_count += 1 + + if args.only_copilot_tagged and issue_copilot_tag != "Yes": + continue + + issue_rows.append( + [ + f"#{issue.number}", + truncate_text(issue.title), + close_time_display, + first_comment_display, + issue_copilot_tag, + ] + ) + + if resolution_times: + mean_time = statistics.mean(resolution_times) + min_time = min(resolution_times) + max_time = max(resolution_times) + else: + mean_time = min_time = max_time = 0 + + print_table( + f"Issue Report (Last {days_lookback} Days)", + ["Repository", "Total New", "Still Open", "Resolved", "Copilot"], + [[repo_name, str(new_count), str(open_count), str(resolved_count), str(issue_copilot_yes_count)]], + ) + print() + print_table( + "Resolution Metrics", + ["Mean", "Min", "Max"], + [[format_duration(mean_time), format_duration(min_time), format_duration(max_time)]], + ) + if not args.summary: + print() + print_table( + "Issue Details", + ["Issue", "Title", "Days to Close", "Time to First Comment", "Copilot"], + issue_rows, + ) + + pull_requests = repo.get_pulls(state="all", sort="created", direction="desc") + + pr_created_count = 0 + pr_merged_count = 0 + pr_abandoned_count = 0 + pr_copilot_yes_count = 0 + pr_owner_counts = Counter() + pr_rows = [] + + for pr in pull_requests: + if pr.created_at < since: + break + + if pr.base is None or pr.base.ref != "main": + continue + + pr_created_count += 1 + time_to_close_display = "Open" + + if pr.state == "closed": + if pr.merged_at is not None: + pr_merged_count += 1 + else: + pr_abandoned_count += 1 + + if pr.closed_at is not None: + close_hours = (pr.closed_at - pr.created_at).total_seconds() / 3600 + time_to_close_display = format_duration(close_hours) + + pr_author = pr.user.login if pr.user else "Unknown" + pr_owner_counts[pr_author] += 1 + pr_author_normalized = pr_author.lower() + pr_copilot_tag = "Yes" if pr_author_normalized in {"dependabot[bot]", "copilot"} else "No" + if pr_copilot_tag != "Yes": + try: + pr_issue = repo.get_issue(pr.number) + pr_copilot_tag = "Yes" if is_copilot_tagged(pr_issue.labels) else "No" + except GithubException: + pr_copilot_tag = "No" + if pr_copilot_tag == "Yes": + pr_copilot_yes_count += 1 + + if args.only_copilot_tagged and pr_copilot_tag != "Yes": + continue + + pr_rows.append( + [ + f"#{pr.number}", + truncate_text(pr.title), + pr_author, + get_first_comment_time_display_for_pr(pr), + time_to_close_display, + pr_copilot_tag, + ] + ) + + print() + print_table( + f"PR Report (Last {days_lookback} Days)", + ["Repository", "PRs Created", "PRs Merged", "PRs Abandoned", "Copilot"], + [[repo_name, str(pr_created_count), str(pr_merged_count), str(pr_abandoned_count), str(pr_copilot_yes_count)]], + ) + + pr_owner_rows = [[owner, str(count)] for owner, count in pr_owner_counts.most_common()] + print() + print_table( + "PR Owner Summary", + ["Author", "PRs"], + pr_owner_rows, + ) + + if not args.summary: + print() + print_table( + "PR Details", + ["PR", "Title", "Author", "Time to First Comment", "Time to Merge/Close", "Copilot"], + pr_rows, + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/tools/health-monitor/internal/stats_export.go b/tools/health-monitor/internal/stats_export.go index dbd8b0200..757e0ed66 100644 --- a/tools/health-monitor/internal/stats_export.go +++ b/tools/health-monitor/internal/stats_export.go @@ -243,7 +243,7 @@ func (se *StatsExporter) checkOutputFile() error { err = se.getNewFile() if err != nil { - log.Err("stats_exporter::checkOutputFile : [%v]") + log.Err("stats_exporter::checkOutputFile : [%v]", err) return err } return nil diff --git a/tools/health-monitor/main.go b/tools/health-monitor/main.go index 317a8bff0..4fea5c286 100644 --- a/tools/health-monitor/main.go +++ b/tools/health-monitor/main.go @@ -119,7 +119,7 @@ func main() { "Blobfuse2 Stats poll interval: %v \n"+ "Health Stats poll interval: %v \n"+ "Cache Path: %v \n"+ - "Max cache size in MB: %v \n", + "Max cache size in MB: %v \n"+ "Output path: %v", hmcommon.Pid, common.TransferPipe, common.PollingPipe, hmcommon.BfsPollInterval, hmcommon.ProcMonInterval, hmcommon.TempCachePath, hmcommon.MaxCacheSize, hmcommon.OutputPath) From deb0acf245da9615cb2d0366d9dabc940a09f41a Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Sat, 21 Mar 2026 13:04:03 +0530 Subject: [PATCH 36/95] Fix release pipeline (#2176) --- blobfuse2-release.yaml | 188 ++++++++++++----------------------------- 1 file changed, 56 insertions(+), 132 deletions(-) diff --git a/blobfuse2-release.yaml b/blobfuse2-release.yaml index c2b677f66..6457e843a 100644 --- a/blobfuse2-release.yaml +++ b/blobfuse2-release.yaml @@ -46,12 +46,12 @@ stages: strategy: matrix: Libfuse: - vmImage: 'ubuntu-18.04' + vmImage: 'ubuntu-20.04' fuselib: 'libfuse-dev' tags: 'fuse2' depends: 'fuse' container: 'test-cnt-ubn-18' - AgentName: "blobfuse-ubuntu18" + AgentName: "blobfuse-ubuntu20" Libfuse3: vmImage: 'ubuntu-20.04' fuselib: 'libfuse3-dev' @@ -282,7 +282,7 @@ stages: timeoutInMinutes: 120 strategy: matrix: - Ubuntu-22: + Ubuntu-20: vmImage: 'Ubuntu-20.04' AgentName: "blobfuse-ubuntu20" pool: @@ -412,127 +412,24 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_0 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - fuse-version: 'fuse3' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo dpkg --info blobfuse2*.deb - sudo dpkg -i blobfuse2*.deb - sudo apt-get install $(fuse-version) build-essential -y - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # Create container for blobfuse and publish it - - script: | - sudo apt-get install $(tags) $(fuselib) -y - chmod 777 *.sh - ./dockerinstall.sh - ./buildcontainer.sh Dockerfile x86_64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 - ./buildcontainer.sh DockerfileMariner mariner-x86_64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 - displayName: "Create and publish container" - workingDirectory: $(work_dir)/docker - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - job: Set_1 timeoutInMinutes: 120 strategy: matrix: - Ubuntu-18: - vmImage: 'Ubuntu-18.04' + Ubuntu-20: + vmImage: 'Ubuntu-20.04' fuselib: 'libfuse-dev' fuse-version: 'fuse' tags: 'fuse2' - container: 'test-cnt-ubn-18' - AgentName: "blobfuse-ubuntu18" - Ubuntu-20: - vmImage: 'Ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + Ubuntu-22: + vmImage: 'Ubuntu-22.04' fuse-version: 'fuse3' fuselib: 'libfuse3-dev' tags: 'fuse3' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" + container: 'test-cnt-ubn-22' + AgentName: "blobfuse-ubuntu22" pool: name: "blobfuse-ubuntu-pool" demands: @@ -594,23 +491,29 @@ stages: - script: | for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - if [ $(AgentName) == "blobfuse-ubuntu18" ]; then - echo "Generating for Debian 9/10" + if [ $(AgentName) == "blobfuse-ubuntu20" ]; then + echo "Generating for Ubuntu-18 and Debian 9/10 for fuse2" f=`ls ./blobfuse2*$(vmImage)*.deb` - cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") - cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") + cp "$f" $(sed 's:Ubuntu-20.04:Ubuntu-18.04:' <<< "$f") + cp "$f" $(sed 's:Ubuntu-20.04:Debian-9.0:' <<< "$f") + cp "$f" $(sed 's:Ubuntu-20.04:Debian-10.0:' <<< "$f") + cp ./blobfuse2*Ubuntu-18*.deb $(Build.ArtifactStagingDirectory) cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) ls -l $(Build.ArtifactStagingDirectory) + rm -rf ./blobfuse2*Ubuntu-18*.deb rm -rf ./blobfuse2*Debian-*.deb else - echo "Generating for Debian 11/12/13" + cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + echo "Generating for Ubuntu-20 and Debian 11/12/13 for fuse3" f=`ls ./blobfuse2*$(vmImage)*.deb` - cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") - cp "$f" $(sed 's:Ubuntu-20.04:Debian-12.0:' <<< "$f") - cp "$f" $(sed 's:Ubuntu-20.04:Debian-13.0:' <<< "$f") + cp "$f" $(sed 's:Ubuntu-22.04:Ubuntu-20.04:' <<< "$f") + cp "$f" $(sed 's:Ubuntu-22.04:Debian-11.0:' <<< "$f") + cp "$f" $(sed 's:Ubuntu-22.04:Debian-12.0:' <<< "$f") + cp "$f" $(sed 's:Ubuntu-22.04:Debian-13.0:' <<< "$f") + cp ./blobfuse2*Ubuntu-20*.deb $(Build.ArtifactStagingDirectory) cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) ls -l $(Build.ArtifactStagingDirectory) + rm -rf ./blobfuse2*Ubuntu-20*.deb rm -rf ./blobfuse2*Debian-*.deb fi displayName: 'Rename Package' @@ -630,6 +533,20 @@ stages: mount_dir: $(mount_dir) temp_dir: $(temp_dir) container: $(container) + + # Create container for blobfuse and publish it + - script: | + if [ $(AgentName) == "blobfuse-ubuntu22" ]; then + sudo apt-get install $(tags) $(fuselib) -y + chmod 777 *.sh + ./dockerinstall.sh + ./buildcontainer.sh Dockerfile x86_64 + ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 + ./buildcontainer.sh DockerfileMariner mariner-x86_64 + ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 + fi + displayName: "Create and publish container" + workingDirectory: $(work_dir)/docker # publishing the artifacts generated - task: PublishBuildArtifacts@1 @@ -1045,12 +962,13 @@ stages: timeoutInMinutes: 120 strategy: matrix: - CentOS-7.9: - agentName: "blobfuse-centos7" - vmImage: 'CentOS-7.0' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-cent-7' + # Commenting as CentOS 7.9 image is not available in the marketplace currently + # CentOS-7.9: + # agentName: "blobfuse-centos7" + # vmImage: 'CentOS-7.0' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-cent-7' CentOS-8.5: agentName: "blobfuse-centos8" vmImage: 'CentOS-8.0' @@ -1128,7 +1046,13 @@ stages: - script: | for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + echo "Generating for CentOS-7.0 for fuse3" + f=`ls ./blobfuse2*$(vmImage)*.rpm` + cp "$f" $(sed 's:CentOS-8.0:CentOS-7.0:' <<< "$f") + cp ./blobfuse2*CentOS-7.0*.rpm $(Build.ArtifactStagingDirectory) + rm -rf ./blobfuse2*CentOS-7.0*.rpm cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + ls -l $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/blobfuse2-signed @@ -1573,8 +1497,8 @@ stages: strategy: matrix: Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" + vmImage: 'Ubuntu-22.04' + AgentName: "blobfuse-ubuntu22" pool: name: "blobfuse-ubuntu-pool" demands: @@ -1605,7 +1529,7 @@ stages: - script: | sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + wget https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt update sudo apt install apt-transport-https -y From 6a0be5cbcb609d4e493731b3a6e332064b1acfca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 11:16:36 +0530 Subject: [PATCH 37/95] Bump actions/checkout from 4 to 6 (#2177) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/blobfuse-buddy.yml | 2 +- .github/workflows/issueMetrics.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/blobfuse-buddy.yml b/.github/workflows/blobfuse-buddy.yml index bd2408f40..0dffd94e2 100644 --- a/.github/workflows/blobfuse-buddy.yml +++ b/.github/workflows/blobfuse-buddy.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Azure Login (OIDC) uses: azure/login@v2 diff --git a/.github/workflows/issueMetrics.yml b/.github/workflows/issueMetrics.yml index 92db71e67..6f20c8339 100644 --- a/.github/workflows/issueMetrics.yml +++ b/.github/workflows/issueMetrics.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Python uses: actions/setup-python@v5 @@ -33,7 +33,7 @@ jobs: python scripts/issueMetrics.py --days 90 > issueMetric.txt - name: Checkout benchmarks branch - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: ref: benchmarks path: benchmarks-branch From b8e5392768a4fff6463d69dce8172d558a19e3f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 01:46:14 +0000 Subject: [PATCH 38/95] Bump actions/setup-python from 5 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/blobfuse-buddy.yml | 2 +- .github/workflows/issueMetrics.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/blobfuse-buddy.yml b/.github/workflows/blobfuse-buddy.yml index 0dffd94e2..6c60687a3 100644 --- a/.github/workflows/blobfuse-buddy.yml +++ b/.github/workflows/blobfuse-buddy.yml @@ -48,7 +48,7 @@ jobs: subscription-id: ${{ secrets.AZURE_MCP_SUBSCRIPTION_ID }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" diff --git a/.github/workflows/issueMetrics.yml b/.github/workflows/issueMetrics.yml index 6f20c8339..495ebe716 100644 --- a/.github/workflows/issueMetrics.yml +++ b/.github/workflows/issueMetrics.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v6 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.x" From cf8882db26e1861205ca50b9de76397d502bedb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 12:33:50 +0530 Subject: [PATCH 39/95] Bump azure/login from 2 to 3 (#2178) --- .github/workflows/blobfuse-buddy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/blobfuse-buddy.yml b/.github/workflows/blobfuse-buddy.yml index 6c60687a3..5d53034a5 100644 --- a/.github/workflows/blobfuse-buddy.yml +++ b/.github/workflows/blobfuse-buddy.yml @@ -41,7 +41,7 @@ jobs: uses: actions/checkout@v6 - name: Azure Login (OIDC) - uses: azure/login@v2 + uses: azure/login@v3 with: client-id: ${{ secrets.AZURE_MCP_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_MCP_TENANT_ID }} From 39191d2f10361ce27f703694ac9b04f93d5fb076 Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Mon, 23 Mar 2026 13:05:53 +0530 Subject: [PATCH 40/95] Release pipeline fix (#2180) --- blobfuse2-release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/blobfuse2-release.yaml b/blobfuse2-release.yaml index 6457e843a..ee43de18a 100644 --- a/blobfuse2-release.yaml +++ b/blobfuse2-release.yaml @@ -1496,9 +1496,9 @@ stages: timeoutInMinutes: 120 strategy: matrix: - Ubuntu-22: - vmImage: 'Ubuntu-22.04' - AgentName: "blobfuse-ubuntu22" + Ubuntu-20: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" pool: name: "blobfuse-ubuntu-pool" demands: @@ -1529,7 +1529,7 @@ stages: - script: | sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt update sudo apt install apt-transport-https -y From 2148334db39c94df90bee031f80dc16e80b1bd73 Mon Sep 17 00:00:00 2001 From: Gauri Lamunion <51212198+gapra-msft@users.noreply.github.com> Date: Mon, 23 Mar 2026 20:05:44 -0700 Subject: [PATCH 41/95] Updated trivy dependency to known safe version (#2181) --- .github/workflows/trivy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 5764b869e..e6d698638 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -45,7 +45,7 @@ jobs: ls -l - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 with: scan-type: fs scan-ref: './blobfuse2' From 5f797a98fdda8826ba126fd3516c51e797df7a32 Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Tue, 24 Mar 2026 11:03:16 +0530 Subject: [PATCH 42/95] Update go mod (#2182) --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 9aed69c6a..824e5d3ae 100644 --- a/go.mod +++ b/go.mod @@ -48,8 +48,8 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.48.0 // indirect - golang.org/x/net v0.51.0 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.42.0 - golang.org/x/text v0.34.0 // indirect + golang.org/x/text v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index 4b8468128..23edac98a 100644 --- a/go.sum +++ b/go.sum @@ -107,14 +107,14 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -128,8 +128,8 @@ golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From cdd32a5c0e99332f7ca26f41ad6d263214eb7dae Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Tue, 24 Mar 2026 11:54:27 +0530 Subject: [PATCH 43/95] Update GitHub service connection (#2183) --- blobfuse2-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blobfuse2-release.yaml b/blobfuse2-release.yaml index ee43de18a..af07f7c7a 100644 --- a/blobfuse2-release.yaml +++ b/blobfuse2-release.yaml @@ -1540,7 +1540,7 @@ stages: # add release tags & push to github - task: GithubRelease@1 inputs: - githubConnection: 'Azure' + githubConnection: 'GitHubRelease' repositoryName: 'Azure/azure-storage-fuse' action: 'edit' From 4e0efc61817d6aafe46f8bf34dd8a72af938c831 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Mar 2026 10:37:59 +0530 Subject: [PATCH 44/95] Bump github.com/montanaflynn/stats from 0.8.2 to 0.9.0 (#2184) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 824e5d3ae..592ccf149 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/golang/mock v1.6.0 - github.com/montanaflynn/stats v0.8.2 + github.com/montanaflynn/stats v0.9.0 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/radovskyb/watcher v1.0.7 github.com/sevlyar/go-daemon v0.1.6 diff --git a/go.sum b/go.sum index 23edac98a..5731a1ef4 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/montanaflynn/stats v0.8.2 h1:52wnefTJnPI5FoHif1DQh2soKRw0yYs+4AVyvtcZCH0= -github.com/montanaflynn/stats v0.8.2/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/montanaflynn/stats v0.9.0 h1:tsBJ0RXwph9BmAuFoCmqGv6e8xa0MENQ8m0ptKq29mQ= +github.com/montanaflynn/stats v0.9.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= From 0652340154f885f5717970d0cbf0252110093ecc Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Wed, 25 Mar 2026 17:16:11 +0530 Subject: [PATCH 45/95] Changelog update post 2.5.3 release (#2187) --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80f43dcdc..33dda6931 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -## 2.5.3 (Unreleased) +## 2.5.4 (Unreleased) +**Features** + +**Bug Fixes** + +## 2.5.3 (2026-03-25) **Features** - Add rate limit functionality for ingress bandwidth (bytes downloaded per second) and operations per second ([PR #2093](https://github.com/Azure/azure-storage-fuse/pull/2093)) From 24036ea6160780679d461b8ddadf550791de1dc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Apr 2026 09:57:30 +0530 Subject: [PATCH 46/95] Bump golang.org/x/sys from 0.42.0 to 0.43.0 (#2190) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 592ccf149..ab034ea64 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/net v0.52.0 // indirect - golang.org/x/sys v0.42.0 + golang.org/x/sys v0.43.0 golang.org/x/text v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index 5731a1ef4..e4c165a81 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 64c3c155e265468a4d5c4167c7373643c20f56fd Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Thu, 9 Apr 2026 10:19:17 +0530 Subject: [PATCH 47/95] Update CODEOWNERS (#2192) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dee0fd1e8..1855c1211 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @vibhansa-msft @souravgupta-msft @syeleti-msft @jainakanksha-msft +* @souravgupta-msft @syeleti-msft @jainakanksha-msft From f57a4cd21fa55a15ad48ab3cf7950a8dfd656a38 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 19:05:28 +0530 Subject: [PATCH 48/95] Bump Go from 1.26.1 to 1.26.2 to fix CVE-2026-33810 (#2193) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: souravgupta-msft <98318303+souravgupta-msft@users.noreply.github.com> --- go.mod | 2 +- go_installer.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index ab034ea64..a053dff4b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Azure/azure-storage-fuse/v2 -go 1.26.1 +go 1.26.2 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 diff --git a/go_installer.sh b/go_installer.sh index 7e128a5d8..9d1a6bae0 100755 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,6 +1,6 @@ #!/bin/bash work_dir=$(echo $1 | sed 's:/*$::') -version="1.26.1" +version="1.26.2" arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` if [ $arch != "arm64" ] From 293e1b4acccaf280e05d278a480205f395db29e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 09:53:33 +0530 Subject: [PATCH 49/95] Bump github.com/Azure/azure-sdk-for-go/sdk/azcore from 1.21.0 to 1.21.1 (#2194) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index a053dff4b..e6050bd4c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Azure/azure-storage-fuse/v2 go 1.26.2 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 @@ -29,7 +29,7 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -48,8 +48,8 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.49.0 // indirect - golang.org/x/net v0.52.0 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/net v0.53.0 // indirect golang.org/x/sys v0.43.0 - golang.org/x/text v0.35.0 // indirect + golang.org/x/text v0.36.0 // indirect ) diff --git a/go.sum b/go.sum index e4c165a81..538c5e41d 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,11 @@ -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= @@ -107,14 +107,14 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -128,8 +128,8 @@ golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From f88601ab22314717df98b18de93d8a997a4cf68a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:52:54 +0530 Subject: [PATCH 50/95] Bump aquasecurity/trivy-action from 0.35.0 to 0.36.0 (#2196) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/trivy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index e6d698638..05f0748c7 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -45,7 +45,7 @@ jobs: ls -l - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: scan-type: fs scan-ref: './blobfuse2' From 90058532b5b15d0f384d412c51e74bd961e8f5bd Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Mon, 4 May 2026 12:04:55 +0530 Subject: [PATCH 51/95] Fix static analysis step - pin golangci-lint to v2.11.0 (#2201) --- blobfuse2-1es_ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/blobfuse2-1es_ci.yaml b/blobfuse2-1es_ci.yaml index 1a6c6ffd6..a8b289ff3 100644 --- a/blobfuse2-1es_ci.yaml +++ b/blobfuse2-1es_ci.yaml @@ -103,7 +103,9 @@ extends: # Code lint checks (Static-analysis) - script: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin + # Pinned to v2.11.0 due to checksum issue in v2.12.1 (bump when upstream fixes it) + # Refer: https://github.com/Azure/azure-storage-fuse/pull/2201 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v2.11.0/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.0 $(go env GOPATH)/bin/golangci-lint --version $(go env GOPATH)/bin/golangci-lint run --build-tags $(tags) > lint.log result=$(cat lint.log | wc -l) From f7b40381ff730b33173271a36f6396c4de92635f Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Mon, 4 May 2026 14:16:32 +0530 Subject: [PATCH 52/95] Fixing url to install golangci-lint (#2202) --- blobfuse2-1es_ci.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/blobfuse2-1es_ci.yaml b/blobfuse2-1es_ci.yaml index a8b289ff3..dfe10e70c 100644 --- a/blobfuse2-1es_ci.yaml +++ b/blobfuse2-1es_ci.yaml @@ -103,9 +103,7 @@ extends: # Code lint checks (Static-analysis) - script: | - # Pinned to v2.11.0 due to checksum issue in v2.12.1 (bump when upstream fixes it) - # Refer: https://github.com/Azure/azure-storage-fuse/pull/2201 - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v2.11.0/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.0 + curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.0 $(go env GOPATH)/bin/golangci-lint --version $(go env GOPATH)/bin/golangci-lint run --build-tags $(tags) > lint.log result=$(cat lint.log | wc -l) From 34c3fccde68da81598fa05f2e964b34811c9cb62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 May 2026 14:55:11 +0530 Subject: [PATCH 53/95] Bump gopkg.in/ini.v1 from 1.67.1 to 1.67.2 (#2200) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e6050bd4c..7312bbbdd 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/vibhansa-msft/tlru v0.0.0-20240410102558-9e708419e21f go.uber.org/atomic v1.11.0 golang.org/x/time v0.15.0 - gopkg.in/ini.v1 v1.67.1 + gopkg.in/ini.v1 v1.67.2 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 538c5e41d..ebe068f1a 100644 --- a/go.sum +++ b/go.sum @@ -141,8 +141,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= -gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= +gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 795a5a48d9332d061a666975006d48a4abec40fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 May 2026 16:16:52 +0530 Subject: [PATCH 54/95] Bump github.com/fsnotify/fsnotify from 1.9.0 to 1.10.0 (#2198) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7312bbbdd..82fbd2557 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda - github.com/fsnotify/fsnotify v1.9.0 + github.com/fsnotify/fsnotify v1.10.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/golang/mock v1.6.0 github.com/montanaflynn/stats v0.9.0 diff --git a/go.sum b/go.sum index ebe068f1a..156a21c9e 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsnotify/fsnotify v1.10.0 h1:Xx/5Ydg9CeBDX/wi4VJqStNtohYjitZhhlHt4h3St1M= +github.com/fsnotify/fsnotify v1.10.0/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= From 80edbd7c0abdb67c6d6c7cae5d7e1065b6c1c984 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 10:34:15 +0530 Subject: [PATCH 55/95] Bump github.com/fsnotify/fsnotify from 1.10.0 to 1.10.1 (#2203) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 82fbd2557..fa9f71085 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda - github.com/fsnotify/fsnotify v1.10.0 + github.com/fsnotify/fsnotify v1.10.1 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/golang/mock v1.6.0 github.com/montanaflynn/stats v0.9.0 diff --git a/go.sum b/go.sum index 156a21c9e..8606941fb 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.10.0 h1:Xx/5Ydg9CeBDX/wi4VJqStNtohYjitZhhlHt4h3St1M= -github.com/fsnotify/fsnotify v1.10.0/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= From 9d552d2707dc63af770b63c0215af1fd8a1bf920 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 May 2026 09:07:44 +0530 Subject: [PATCH 56/95] Replace on-demand CI workflow with comment-triggered PR mirror for ADO (#2204) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: syeleti-msft <172502481+syeleti-msft@users.noreply.github.com> Co-authored-by: syeleti-msft Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/pr-mirror-cleanup.yml | 69 ++++++++++++ .github/workflows/pr-mirror.yml | 141 ++++++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 .github/workflows/pr-mirror-cleanup.yml create mode 100644 .github/workflows/pr-mirror.yml diff --git a/.github/workflows/pr-mirror-cleanup.yml b/.github/workflows/pr-mirror-cleanup.yml new file mode 100644 index 000000000..56a5c4fe9 --- /dev/null +++ b/.github/workflows/pr-mirror-cleanup.yml @@ -0,0 +1,69 @@ +# Daily cleanup of `pr-mirror/` branches created by pr-mirror.yml. +# +# A mirror branch is deleted when: +# * the corresponding pull request is closed or merged, OR +# * the branch tip no longer matches the PR's current head SHA (the PR was +# force-pushed and not re-mirrored). +# +# This workflow uses no secrets and only mutates `pr-mirror/**` branches. + +name: Cleanup PR mirror branches + +on: + schedule: + - cron: '17 4 * * *' + workflow_dispatch: + +permissions: + contents: write + pull-requests: read + +jobs: + cleanup: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Delete stale pr-mirror/* branches + uses: actions/github-script@v7 + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const prefix = 'pr-mirror/'; + + const branches = await github.paginate( + github.rest.repos.listBranches, + { owner, repo, per_page: 100 } + ); + + for (const b of branches) { + if (!b.name.startsWith(prefix)) continue; + const num = Number(b.name.substring(prefix.length)); + if (!Number.isInteger(num) || num <= 0) continue; + + let reason = null; + try { + const { data: pr } = await github.rest.pulls.get({ + owner, repo, pull_number: num + }); + if (pr.state !== 'open') { + reason = `PR #${num} is ${pr.state}`; + } else if (pr.head.sha !== b.commit.sha) { + reason = `PR #${num} head ${pr.head.sha.substring(0,7)} != mirror tip ${b.commit.sha.substring(0,7)}`; + } + } catch (e) { + if (e.status === 404) { + reason = `PR #${num} not found`; + } else { + core.warning(`Skipping ${b.name}: ${e.message}`); + continue; + } + } + + if (reason) { + core.info(`Deleting ${b.name} (${reason})`); + await github.rest.git.deleteRef({ + owner, repo, ref: `heads/${b.name}` + }); + } + } diff --git a/.github/workflows/pr-mirror.yml b/.github/workflows/pr-mirror.yml new file mode 100644 index 000000000..addeffc7b --- /dev/null +++ b/.github/workflows/pr-mirror.yml @@ -0,0 +1,141 @@ +# Mirror an external PR's head commit to a branch in this repository so that +# the Azure DevOps `blobfuse2-1es_ci` pipeline (which cannot run on PRs from +# forks) can be queued manually by a maintainer. +# +# Trigger: a maintainer comments `/mirror` on a pull request. +# Authorize: the comment author must have association OWNER or MEMBER. +# Action: fetch the PR head SHA via `refs/pull//head` (no fork code is +# ever executed in this workflow) and force-push it to +# `refs/heads/pr-mirror/` in this repository. +# Result: a maintainer can open the ADO pipeline UI, choose +# `pr-mirror/` as the source branch, and click "Run". +# +# Security notes: +# * This workflow uses no repository secrets and never checks out fork code. +# * Branch protection / rulesets should restrict who can push to +# `pr-mirror/**` to the GitHub Actions bot only, and these branches must +# not be allowed as merge sources into `main`. +# * Pin `blobfuse2-1es_ci` to load its YAML from a fixed ref (e.g. `main`) +# so that fork-authored changes to the pipeline file cannot influence the +# manually queued run. Otherwise diff the pipeline YAML and any scripts it +# invokes before queueing. + +name: Mirror PR for ADO CI + +on: + issue_comment: + types: [created] + +permissions: + contents: write + pull-requests: write + issues: write + statuses: write + +jobs: + mirror: + name: Mirror PR head to pr-mirror/ + if: >- + github.event.issue.pull_request != null && + github.event.comment.body == '/mirror' && + (github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'MEMBER') + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: React to triggering comment + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'rocket' + }); + + - name: Resolve PR head SHA + id: pr + uses: actions/github-script@v7 + with: + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + core.setOutput('sha', pr.data.head.sha); + core.setOutput('number', String(pr.data.number)); + core.setOutput('state', pr.data.state); + + - name: Refuse if PR is closed + if: steps.pr.outputs.state != 'open' + run: | + echo "::error::PR #${{ steps.pr.outputs.number }} is not open; refusing to mirror." + exit 1 + + - name: Checkout repo (no fork code) + uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Fetch PR head and push to mirror branch + env: + PR_NUMBER: ${{ steps.pr.outputs.number }} + PR_SHA: ${{ steps.pr.outputs.sha }} + run: | + set -euo pipefail + # GitHub exposes every PR on the upstream repo as refs/pull//head. + # Fetching from `origin` keeps the operation entirely within this + # repository and avoids contacting the fork directly. + git fetch --no-tags origin "pull/${PR_NUMBER}/head" + fetched_sha="$(git rev-parse FETCH_HEAD)" + if [ "${fetched_sha}" != "${PR_SHA}" ]; then + echo "::error::Fetched SHA ${fetched_sha} does not match PR head ${PR_SHA}." + exit 1 + fi + # Populate/update the local remote-tracking ref for the destination + # branch so --force-with-lease has an expected old value on reruns. + if git ls-remote --exit-code --heads origin "pr-mirror/${PR_NUMBER}" >/dev/null 2>&1; then + git fetch --no-tags origin \ + "refs/heads/pr-mirror/${PR_NUMBER}:refs/remotes/origin/pr-mirror/${PR_NUMBER}" + fi + git push --force-with-lease origin \ + "${PR_SHA}:refs/heads/pr-mirror/${PR_NUMBER}" + + - name: Comment with mirror branch details + uses: actions/github-script@v7 + with: + script: | + const sha = '${{ steps.pr.outputs.sha }}'; + const num = '${{ steps.pr.outputs.number }}'; + const branch = `pr-mirror/${num}`; + const shortSha = sha.substring(0, 7); + const branchUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/tree/${branch}`; + const body = [ + `🔁 Mirrored \`${shortSha}\` to [\`${branch}\`](${branchUrl}).`, + ``, + `Maintainers: queue the **blobfuse2-1es_ci** pipeline in Azure DevOps`, + `against branch \`${branch}\` to run the credentialed CI.`, + ``, + `Re-comment \`/mirror\` after any new push to refresh this branch.`, + ].join('\n'); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: Number(num), + body + }); + + - name: Set pending commit status + uses: actions/github-script@v7 + with: + script: | + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ steps.pr.outputs.sha }}', + state: 'pending', + context: 'pr-mirror', + description: 'Mirrored to pr-mirror/${{ steps.pr.outputs.number }}; awaiting ADO run' + }); From 6857ad8278408b80cb3c32777bc5f2a03d056f47 Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Thu, 7 May 2026 14:10:37 +0530 Subject: [PATCH 57/95] Trigger ci automcatically for the branches with prefix pr-mirror/* (#2205) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- blobfuse2-1es_ci.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/blobfuse2-1es_ci.yaml b/blobfuse2-1es_ci.yaml index dfe10e70c..22cab9d07 100644 --- a/blobfuse2-1es_ci.yaml +++ b/blobfuse2-1es_ci.yaml @@ -1,5 +1,8 @@ -# Do not trigger this pipeline automatically -trigger: none +# Run automatically for pr-mirror/* branches +trigger: + branches: + include: + - pr-mirror/* # Execute this for every PR on main pr: From c9871f8d94424b6c74937d4c93ec4452bf6acc97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 May 2026 10:54:17 +0530 Subject: [PATCH 58/95] Bump actions/github-script from 7 to 9 (#2207) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pr-mirror-cleanup.yml | 2 +- .github/workflows/pr-mirror.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-mirror-cleanup.yml b/.github/workflows/pr-mirror-cleanup.yml index 56a5c4fe9..d1709371e 100644 --- a/.github/workflows/pr-mirror-cleanup.yml +++ b/.github/workflows/pr-mirror-cleanup.yml @@ -24,7 +24,7 @@ jobs: timeout-minutes: 10 steps: - name: Delete stale pr-mirror/* branches - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | const owner = context.repo.owner; diff --git a/.github/workflows/pr-mirror.yml b/.github/workflows/pr-mirror.yml index addeffc7b..b47155b67 100644 --- a/.github/workflows/pr-mirror.yml +++ b/.github/workflows/pr-mirror.yml @@ -44,7 +44,7 @@ jobs: timeout-minutes: 10 steps: - name: React to triggering comment - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | await github.rest.reactions.createForIssueComment({ @@ -56,7 +56,7 @@ jobs: - name: Resolve PR head SHA id: pr - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | const pr = await github.rest.pulls.get({ @@ -104,7 +104,7 @@ jobs: "${PR_SHA}:refs/heads/pr-mirror/${PR_NUMBER}" - name: Comment with mirror branch details - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | const sha = '${{ steps.pr.outputs.sha }}'; @@ -128,7 +128,7 @@ jobs: }); - name: Set pending commit status - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | await github.rest.repos.createCommitStatus({ From 7adaed51fee4dab880ca7de8ffdb0092a07e98d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 10:04:08 +0530 Subject: [PATCH 59/95] Bump golang.org/x/sys from 0.43.0 to 0.44.0 (#2210) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fa9f71085..ec701cb3b 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.50.0 // indirect golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 + golang.org/x/sys v0.44.0 golang.org/x/text v0.36.0 // indirect ) diff --git a/go.sum b/go.sum index 8606941fb..f03c44263 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From d85c6e2d7b27be1c13f67026ddd2fc7d4c723367 Mon Sep 17 00:00:00 2001 From: imddanish Date: Mon, 11 May 2026 12:30:49 +0530 Subject: [PATCH 60/95] Fix duplicated prefix in CPK block blob attribute lookup for subdirectory mounts (#2199) Co-authored-by: MD Danish --- CHANGELOG.md | 1 + component/azstorage/block_blob.go | 2 +- component/azstorage/block_blob_test.go | 70 ++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33dda6931..6aeb5302e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ **Features** **Bug Fixes** +- Fix CPK-encrypted blob attribute lookup duplicating the prefix path when subdirectory is configured ([PR #2199](https://github.com/Azure/azure-storage-fuse/pull/2199)) ## 2.5.3 (2026-03-25) **Features** diff --git a/component/azstorage/block_blob.go b/component/azstorage/block_blob.go index b710d82f8..be5e7f642 100644 --- a/component/azstorage/block_blob.go +++ b/component/azstorage/block_blob.go @@ -702,7 +702,7 @@ func (bb *BlockBlob) processBlobItems(blobItems []*container.BlobItem) ([]*inter func (bb *BlockBlob) getBlobAttr(blobInfo *container.BlobItem) (*internal.ObjAttr, error) { if blobInfo.Properties.CustomerProvidedKeySHA256 != nil && *blobInfo.Properties.CustomerProvidedKeySHA256 != "" { log.Trace("BlockBlob::List : blob is encrypted with customer provided key so fetching metadata explicitly using REST") - return bb.getAttrUsingRest(*blobInfo.Name) + return bb.getAttrUsingRest(removePrefixPath(bb.Config.prefixPath, *blobInfo.Name)) } mode, err := bb.getFileMode(blobInfo.Properties.Permissions) if err != nil { diff --git a/component/azstorage/block_blob_test.go b/component/azstorage/block_blob_test.go index 75edbc0c4..329eb6489 100644 --- a/component/azstorage/block_blob_test.go +++ b/component/azstorage/block_blob_test.go @@ -2094,6 +2094,76 @@ func (s *blockBlobTestSuite) TestGetAttrDirWithCPKEnabled() { s.assert.True(checkMetadata(props.Metadata, folderKey, "true")) } +func (s *blockBlobTestSuite) TestGetAttrFileWithCPKEnabledAndPrefixPath() { + defer s.cleanupTest() + CPKEncryptionKey, CPKEncryptionKeySHA256 := generateCPKInfo() + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n cpk-enabled: true\n cpk-encryption-key: %s\n cpk-encryption-key-sha256: %s\n account-key: %s\n mode: key\n container: %s\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, CPKEncryptionKey, CPKEncryptionKeySHA256, storageTestConfigurationParameters.BlockKey, s.container) + + s.tearDownTestHelper(false) + s.setupTestHelper(config, s.container, false) + + // Create a subdirectory to use as a prefix path and a file inside it + prefix := generateDirectoryName() + err := s.az.CreateDir(internal.CreateDirOptions{Name: prefix}) + s.assert.NoError(err) + + fileName := generateFileName() + filePath := prefix + "/" + fileName + _, err = s.az.CreateFile(internal.CreateFileOptions{Name: filePath}) + s.assert.NoError(err) + + // Set the prefix path to the subdirectory + _ = s.az.storage.SetPrefixPath(prefix) + + // GetAttr should resolve the file correctly without duplicating the prefix path + props, err := s.az.GetAttr(internal.GetAttrOptions{Name: fileName}) + s.assert.NoError(err) + s.assert.NotNil(props) + s.assert.Equal(fileName, props.Path) + s.assert.False(props.IsDir()) +} + +func (s *blockBlobTestSuite) TestReadDirWithCPKEnabledAndPrefixPath() { + defer s.cleanupTest() + CPKEncryptionKey, CPKEncryptionKeySHA256 := generateCPKInfo() + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n cpk-enabled: true\n cpk-encryption-key: %s\n cpk-encryption-key-sha256: %s\n account-key: %s\n mode: key\n container: %s\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, CPKEncryptionKey, CPKEncryptionKeySHA256, storageTestConfigurationParameters.BlockKey, s.container) + + s.tearDownTestHelper(false) + s.setupTestHelper(config, s.container, false) + + // Create a subdirectory to use as a prefix path with files inside it + prefix := generateDirectoryName() + err := s.az.CreateDir(internal.CreateDirOptions{Name: prefix}) + s.assert.NoError(err) + + fileName := generateFileName() + filePath := prefix + "/" + fileName + _, err = s.az.CreateFile(internal.CreateFileOptions{Name: filePath}) + s.assert.NoError(err) + + // Set the prefix path to the subdirectory + _ = s.az.storage.SetPrefixPath(prefix) + + // ReadDir should list the file correctly without duplicating the prefix path + entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: "/"}) + s.assert.NoError(err) + s.assert.NotEmpty(entries) + + // Verify the file path does not have a duplicated prefix + found := false + for _, entry := range entries { + if entry.Name == fileName { + found = true + s.assert.Equal(fileName, entry.Path) + s.assert.False(entry.IsDir()) + break + } + } + s.assert.True(found, "Expected file not found in ReadDir results") +} + func (s *blockBlobTestSuite) TestGetAttrFile() { defer s.cleanupTest() vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", From 152af28c37fc45094632ac2497a3bf641266631b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 10:42:09 +0530 Subject: [PATCH 61/95] Bump github.com/Azure/azure-sdk-for-go/sdk/storage/azblob from 1.6.4 to 1.7.0 (#2217) --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index ec701cb3b..233c9b320 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.2 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda github.com/fsnotify/fsnotify v1.10.1 @@ -30,7 +30,7 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect @@ -48,8 +48,8 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/net v0.53.0 // indirect + golang.org/x/crypto v0.51.0 // indirect + golang.org/x/net v0.54.0 // indirect golang.org/x/sys v0.44.0 - golang.org/x/text v0.36.0 // indirect + golang.org/x/text v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index f03c44263..5b4dfe99a 100644 --- a/go.sum +++ b/go.sum @@ -8,14 +8,14 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 h1:BM85pSYlVYQHdq00nxyPoOkyLF5NArJG3bOsrmbwr4k= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0/go.mod h1:QYjP2cB7ZYtS/8jAbE0VSBZde/tjExqGjp+8JY6/+ts= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 h1:7QtoGxKm6mPhsWzEZtrn3tQF1hmMMZblngnqNoE61I8= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4/go.mod h1:juYrzH1q6A+g9ZZbGh0OmjS7zaMq3rFDrPhVnYSgFMA= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 h1:4iB+IesclUXdP0ICgAabvq2FYLXrJWKx1fJQ+GxSo3Y= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNPJ3W56nxOKb4164LEw094U0C8zYQM8mQU= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda/go.mod h1:2CaSFTh2ph9ymS6goiOKIBdfhwWUVsX4nQ5QjIYFHHs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -107,14 +107,14 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -128,8 +128,8 @@ golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 5a860acddfac4e3d3f4125afbe60e1402d7845df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 11:54:17 +0530 Subject: [PATCH 62/95] Bump github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake from 1.4.4 to 1.5.0 (#2219) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 233c9b320..fdc517837 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 + github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.5.0 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda github.com/fsnotify/fsnotify v1.10.1 github.com/go-viper/mapstructure/v2 v2.5.0 diff --git a/go.sum b/go.sum index 5b4dfe99a..21e19a6ca 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 h1:BM85pSYlVYQHdq00nxyPoOkyLF5NArJG3bOsrmbwr4k= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0/go.mod h1:QYjP2cB7ZYtS/8jAbE0VSBZde/tjExqGjp+8JY6/+ts= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 h1:7QtoGxKm6mPhsWzEZtrn3tQF1hmMMZblngnqNoE61I8= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4/go.mod h1:juYrzH1q6A+g9ZZbGh0OmjS7zaMq3rFDrPhVnYSgFMA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.5.0 h1:xQ+0Of0JPDx03Fgq64eblLeFhAKv3ZtxT5Eg5QxkZFU= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.5.0/go.mod h1:SSAgdUeg1SlBmOHGAsmLK5m14YvdjLoJB3tZi51zukU= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 85a84cf6487b9a6b4a4b4d589fc6b4e466074032 Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Mon, 18 May 2026 14:33:32 +0530 Subject: [PATCH 63/95] Replace interface{}->any to follow newer syntax (#2211) --- component/libfuse/libfuse2_handler.go | 16 ++++++++-------- component/libfuse/libfuse_handler.go | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/component/libfuse/libfuse2_handler.go b/component/libfuse/libfuse2_handler.go index 541162fd1..c7d2b6f64 100644 --- a/component/libfuse/libfuse2_handler.go +++ b/component/libfuse/libfuse2_handler.go @@ -446,7 +446,7 @@ func libfuse_mkdir(path *C.char, mode C.mode_t) C.int { } } - libfuseStatsCollector.PushEvents(createDir, name, map[string]interface{}{md: fs.FileMode(uint32(mode) & 0xffffffff)}) + libfuseStatsCollector.PushEvents(createDir, name, map[string]any{md: fs.FileMode(uint32(mode) & 0xffffffff)}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, createDir, (int64)(1)) return 0 @@ -625,7 +625,7 @@ func libfuse_create(path *C.char, mode C.mode_t, fi *C.fuse_file_info_t) C.int { log.Trace("Libfuse::libfuse2_create : %s, handle %d", name, handle.ID) fi.fh = C.uint64_t(uintptr(unsafe.Pointer(ret_val))) - libfuseStatsCollector.PushEvents(createFile, name, map[string]interface{}{md: fs.FileMode(uint32(mode) & 0xffffffff)}) + libfuseStatsCollector.PushEvents(createFile, name, map[string]any{md: fs.FileMode(uint32(mode) & 0xffffffff)}) // increment open file handles count libfuseStatsCollector.UpdateStats(stats_manager.Increment, openHandles, (int64)(1)) @@ -879,7 +879,7 @@ func libfuse2_truncate(path *C.char, off C.off_t) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(truncateFile, name, map[string]interface{}{size: int64(off)}) + libfuseStatsCollector.PushEvents(truncateFile, name, map[string]any{size: int64(off)}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, truncateFile, (int64)(1)) return 0 @@ -965,7 +965,7 @@ func libfuse2_rename(src *C.char, dst *C.char) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(renameDir, srcPath, map[string]interface{}{source: srcPath, dest: dstPath}) + libfuseStatsCollector.PushEvents(renameDir, srcPath, map[string]any{source: srcPath, dest: dstPath}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, renameDir, (int64)(1)) } else { @@ -980,7 +980,7 @@ func libfuse2_rename(src *C.char, dst *C.char) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(renameFile, srcPath, map[string]interface{}{source: srcPath, dest: dstPath}) + libfuseStatsCollector.PushEvents(renameFile, srcPath, map[string]any{source: srcPath, dest: dstPath}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, renameFile, (int64)(1)) } @@ -1006,7 +1006,7 @@ func libfuse_symlink(target *C.char, link *C.char) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(createLink, name, map[string]interface{}{trgt: targetPath}) + libfuseStatsCollector.PushEvents(createLink, name, map[string]any{trgt: targetPath}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, createLink, (int64)(1)) return 0 @@ -1038,7 +1038,7 @@ func libfuse_readlink(path *C.char, buf *C.char, size C.size_t) C.int { copy(data[:size-1], targetPath) data[len(targetPath)] = 0 - libfuseStatsCollector.PushEvents(readLink, name, map[string]interface{}{trgt: targetPath}) + libfuseStatsCollector.PushEvents(readLink, name, map[string]any{trgt: targetPath}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, readLink, (int64)(1)) return 0 @@ -1091,7 +1091,7 @@ func libfuse2_chmod(path *C.char, mode C.mode_t) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(chmod, name, map[string]interface{}{md: fs.FileMode(uint32(mode) & 0xffffffff)}) + libfuseStatsCollector.PushEvents(chmod, name, map[string]any{md: fs.FileMode(uint32(mode) & 0xffffffff)}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, chmod, (int64)(1)) return 0 diff --git a/component/libfuse/libfuse_handler.go b/component/libfuse/libfuse_handler.go index ff1e6ce1a..f22f69243 100644 --- a/component/libfuse/libfuse_handler.go +++ b/component/libfuse/libfuse_handler.go @@ -450,7 +450,7 @@ func libfuse_mkdir(path *C.char, mode C.mode_t) C.int { } } - libfuseStatsCollector.PushEvents(createDir, name, map[string]interface{}{md: fs.FileMode(uint32(mode) & 0xffffffff)}) + libfuseStatsCollector.PushEvents(createDir, name, map[string]any{md: fs.FileMode(uint32(mode) & 0xffffffff)}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, createDir, (int64)(1)) return 0 @@ -663,7 +663,7 @@ func libfuse_create(path *C.char, mode C.mode_t, fi *C.fuse_file_info_t) C.int { log.Trace("Libfuse::libfuse_create : %s, handle %d", name, handle.ID) fi.fh = C.uint64_t(uintptr(unsafe.Pointer(ret_val))) - libfuseStatsCollector.PushEvents(createFile, name, map[string]interface{}{md: fs.FileMode(uint32(mode) & 0xffffffff)}) + libfuseStatsCollector.PushEvents(createFile, name, map[string]any{md: fs.FileMode(uint32(mode) & 0xffffffff)}) // increment open file handles count libfuseStatsCollector.UpdateStats(stats_manager.Increment, openHandles, (int64)(1)) @@ -943,7 +943,7 @@ func libfuse_truncate(path *C.char, off C.off_t, fi *C.fuse_file_info_t) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(truncateFile, name, map[string]interface{}{size: int64(off)}) + libfuseStatsCollector.PushEvents(truncateFile, name, map[string]any{size: int64(off)}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, truncateFile, (int64)(1)) return 0 @@ -1042,7 +1042,7 @@ func libfuse_rename(src *C.char, dst *C.char, flags C.uint) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(renameDir, srcPath, map[string]interface{}{source: srcPath, dest: dstPath}) + libfuseStatsCollector.PushEvents(renameDir, srcPath, map[string]any{source: srcPath, dest: dstPath}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, renameDir, (int64)(1)) } else { @@ -1057,7 +1057,7 @@ func libfuse_rename(src *C.char, dst *C.char, flags C.uint) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(renameFile, srcPath, map[string]interface{}{source: srcPath, dest: dstPath}) + libfuseStatsCollector.PushEvents(renameFile, srcPath, map[string]any{source: srcPath, dest: dstPath}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, renameFile, (int64)(1)) } @@ -1083,7 +1083,7 @@ func libfuse_symlink(target *C.char, link *C.char) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(createLink, name, map[string]interface{}{trgt: targetPath}) + libfuseStatsCollector.PushEvents(createLink, name, map[string]any{trgt: targetPath}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, createLink, (int64)(1)) return 0 @@ -1115,7 +1115,7 @@ func libfuse_readlink(path *C.char, buf *C.char, size C.size_t) C.int { copy(data[:size-1], targetPath) data[len(targetPath)] = 0 - libfuseStatsCollector.PushEvents(readLink, name, map[string]interface{}{trgt: targetPath}) + libfuseStatsCollector.PushEvents(readLink, name, map[string]any{trgt: targetPath}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, readLink, (int64)(1)) return 0 @@ -1168,7 +1168,7 @@ func libfuse_chmod(path *C.char, mode C.mode_t, fi *C.fuse_file_info_t) C.int { return -C.EIO } - libfuseStatsCollector.PushEvents(chmod, name, map[string]interface{}{md: fs.FileMode(uint32(mode) & 0xffffffff)}) + libfuseStatsCollector.PushEvents(chmod, name, map[string]any{md: fs.FileMode(uint32(mode) & 0xffffffff)}) libfuseStatsCollector.UpdateStats(stats_manager.Increment, chmod, (int64)(1)) return 0 From 8bf5ff6a5ccabbb2ab82ddb4fc412bc851479667 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 09:09:25 +0530 Subject: [PATCH 64/95] Bump golang.org/x/sys from 0.44.0 to 0.45.0 (#2225) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fdc517837..318d3e13b 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.51.0 // indirect golang.org/x/net v0.54.0 // indirect - golang.org/x/sys v0.44.0 + golang.org/x/sys v0.45.0 golang.org/x/text v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index 21e19a6ca..799128f0d 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From b5b61a49e2f8949ec9962c4ab726218e8d289270 Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:29:08 +0530 Subject: [PATCH 65/95] FIPS compliance (#2226) --- CHANGELOG.md | 1 + azure-pipeline-templates/build-release.yml | 16 ++-- azure-pipeline-templates/build.yml | 23 ++--- build.sh | 11 ++- common/types.go | 2 +- go.mod | 2 +- go_installer.sh | 98 ++++++++++++++++++++-- 7 files changed, 126 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aeb5302e..e88284004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## 2.5.4 (Unreleased) **Features** +- Make Blobfuse2 binary FIPS compliant by building with the Microsoft Go toolchain (`systemcrypto` GOEXPERIMENT) and `CGO_ENABLED=1`, routing all `crypto/*` calls through the system OpenSSL FIPS provider ([PR #2226](https://github.com/Azure/azure-storage-fuse/pull/2226)) **Bug Fixes** - Fix CPK-encrypted blob attribute lookup duplicating the prefix path when subdirectory is configured ([PR #2199](https://github.com/Azure/azure-storage-fuse/pull/2199)) diff --git a/azure-pipeline-templates/build-release.yml b/azure-pipeline-templates/build-release.yml index 0622e5fac..883a68846 100644 --- a/azure-pipeline-templates/build-release.yml +++ b/azure-pipeline-templates/build-release.yml @@ -29,12 +29,16 @@ steps: displayName: "Installing Blobfuse2 Dependencies" # Build the binary - - task: Go@0 - inputs: - command: 'build' - arguments: "-tags ${{ parameters.tags }} -o blobfuse2" - workingDirectory: ${{ parameters.work_dir }} - displayName: 'Building Blobfuse2' + # build.sh sets CGO_ENABLED=1 and GOTOOLCHAIN=local, but FIPS compliance + # also requires CI to install the Microsoft Go toolchain via go_installer.sh + # first, because GOTOOLCHAIN=local uses that locally installed toolchain. + # Keeping the build flags and required toolchain aligned avoids drift between + # local dev and CI/release builds. + - script: | + chmod 755 build.sh + ./build.sh ${{ parameters.tags }} + workingDirectory: ${{ parameters.work_dir }} + displayName: "Building Blobfuse2" # Verifying whether built binary is correct - script: | diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index cecaa242a..5d1347b77 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -83,19 +83,22 @@ steps: displayName: "Go Get" # Building blobfuse2 binary - - task: Go@0 - inputs: - command: 'build' - arguments: "-tags $(tags) -o blobfuse2" - workingDirectory: $(WORK_DIR) + # build.sh centralizes CI build settings such as CGO_ENABLED=1 and + # GOTOOLCHAIN=local, but the systemcrypto marker required for FIPS + # compliance also depends on using the Microsoft Go toolchain and the + # appropriate GOEXPERIMENT behavior. Keep these build settings centralized + # in build.sh so CI matches local dev exactly. + - script: | + chmod 755 build.sh + ./build.sh $(tags) + workingDirectory: $(WORK_DIR) displayName: "Go Build" # Building bfusemon binary - - task: Go@0 - inputs: - command: 'build' - arguments: "-tags $(tags) -o bfusemon ./tools/health-monitor/" - workingDirectory: $(WORK_DIR) + - script: | + chmod 755 build.sh + ./build.sh health + workingDirectory: $(WORK_DIR) condition: eq('${{ parameters.skip_bfusemon }}', false) displayName: "Go Build bfusemon" diff --git a/build.sh b/build.sh index 8e0e5cb87..63a09808a 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,14 @@ #!/bin/bash echo "Using Go - $(go version)" + +# FIPS compliance: blobfuse2 ships in the Linux package feed that AKS Blob +# CSI consumes, so the binary must be built with the Microsoft Go toolchain +# (systemcrypto GOEXPERIMENT) and CGO enabled. Pin the toolchain to the +# locally installed Go so a stray `go` directive bump cannot trigger a +# silent download of an upstream non-FIPS toolchain. +export CGO_ENABLED=1 +export GOTOOLCHAIN=local + if [ "$1" == "fuse2" ] then # Build blobfuse2 with fuse2 @@ -16,4 +25,4 @@ else rm -rf blobfuse2 rm -rf azure-storage-fuse go build -o blobfuse2 -fi \ No newline at end of file +fi diff --git a/common/types.go b/common/types.go index 858a4c923..4047fb35f 100644 --- a/common/types.go +++ b/common/types.go @@ -47,7 +47,7 @@ import ( // Standard config default values const ( - blobfuse2Version_ = "2.5.3" + blobfuse2Version_ = "2.5.4" DefaultMaxLogFileSize = 512 DefaultLogFileCount = 10 diff --git a/go.mod b/go.mod index 318d3e13b..c17f2afd1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Azure/azure-storage-fuse/v2 -go 1.26.2 +go 1.26.3 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 diff --git a/go_installer.sh b/go_installer.sh index 9d1a6bae0..47aca20af 100755 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,16 +1,98 @@ #!/bin/bash -work_dir=$(echo $1 | sed 's:/*$::') -version="1.26.2" -arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` +# Fail fast on any error, unset variable, or pipe failure so a broken +# download/extract cannot leave the agent in a half-installed state. +set -euo pipefail -if [ $arch != "arm64" ] +work_dir=$(echo "$1" | sed 's:/*$::') + +# Microsoft build of Go (FIPS-capable; ships systemcrypto GOEXPERIMENT). +# Pinned to a specific patch version for reproducible builds. Override by +# exporting GO_VERSION before run (either a specific version like "1.26.3" +# or a major.minor stream like "1.26" which the aka.ms redirector resolves +# to the latest patch in that line). +version="${GO_VERSION:-1.26.3}" +arch=$(hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev) + +if [ "$arch" != "arm64" ] then arch="amd64" fi -echo "Installing on : " $arch " Version : " $version -wget "https://golang.org/dl/go$version.linux-$arch.tar.gz" -P "$work_dir" -sudo rm -rf /usr/local/go -sudo tar -C /usr/local -xzf "$work_dir"/go"$version".linux-$arch.tar.gz +echo "Installing Microsoft Go (FIPS) on : $arch Stream : $version" +tarball="go${version}.linux-${arch}.tar.gz" + +# TLS-verified download from the Microsoft-controlled aka.ms redirector. +# --tries / --timeout avoid hanging the pipeline on transient network issues. +wget --tries=3 --timeout=60 -O "$work_dir/$tarball" \ + "https://aka.ms/golang/release/latest/${tarball}" + +# Verify the tarball against the official SHA256 sidecar published next to +# the release. This guards against a compromised redirector, a corrupted +# upstream artifact, or a man-in-the-middle that has TLS but not the +# checksum. The sidecar lists the resolved release filename (e.g. +# "go1.26.3-20260508.1.linux-amd64.tar.gz"), so we extract just the hash +# and compare it against the locally computed hash of the downloaded file. +wget --tries=3 --timeout=60 -O "$work_dir/$tarball.sha256" \ + "https://aka.ms/golang/release/latest/${tarball}.sha256" + +expected_sha=$(awk '{print $1}' "$work_dir/$tarball.sha256") +actual_sha=$(sha256sum "$work_dir/$tarball" | awk '{print $1}') +if [ "$expected_sha" != "$actual_sha" ]; then + echo "ERROR: SHA256 mismatch for $tarball" >&2 + echo " expected: $expected_sha" >&2 + echo " actual: $actual_sha" >&2 + rm -f "$work_dir/$tarball" "$work_dir/$tarball.sha256" + exit 1 +fi +echo "SHA256 verified: $actual_sha" +rm "$work_dir/$tarball.sha256" + +# Stage the new toolchain in /usr/local/go.new before touching the existing +# install. Keeping the staging dir on the same filesystem as /usr/local/go +# ensures the final `mv` is atomic, so a failed tar / power loss / partial +# extract cannot leave the agent without a working Go. +NEW_GOROOT="/usr/local/go.new" +OLD_GOROOT="/usr/local/go.old" +sudo rm -rf "$NEW_GOROOT" "$OLD_GOROOT" +sudo mkdir -p "$NEW_GOROOT" + +# Clean up the staging dir if anything below fails before we swap. +trap 'sudo rm -rf "$NEW_GOROOT"' ERR + +# Extract into the staging dir. --strip-components=1 drops the tarball's +# top-level "go/" so the contents land directly under NEW_GOROOT. +sudo tar -C "$NEW_GOROOT" --strip-components=1 -xzf "$work_dir/$tarball" + +# Validate the staged toolchain BEFORE swapping. If any of these checks +# fail, the existing /usr/local/go is left intact. +"$NEW_GOROOT/bin/go" version + +# Verify this is the Microsoft build of Go, not upstream. The MS fork +# ships a MICROSOFT_REVISION file at GOROOT that upstream Go does not. +# This is the definitive FIPS-lineage check: without MS Go, the +# systemcrypto GOEXPERIMENT is unavailable and binaries built here will +# not carry the microsoft_systemcrypto marker required for FIPS. +if [ ! -f "$NEW_GOROOT/MICROSOFT_REVISION" ]; then + echo "ERROR: $NEW_GOROOT/MICROSOFT_REVISION not found." >&2 + echo " The installed toolchain is not the Microsoft build of Go." >&2 + echo " FIPS-compliant binaries cannot be produced. Aborting." >&2 + exit 1 +fi +echo "Microsoft Go revision: $(cat "$NEW_GOROOT/MICROSOFT_REVISION")" + +# Atomic swap: move the existing install aside, then move the new one in. +# Both renames are within /usr/local (same filesystem) so each is atomic +# on Linux. After the swap, prune the old install. The trap above is +# cleared since the staging dir has been moved into place. +if [ -d /usr/local/go ]; then + sudo mv /usr/local/go "$OLD_GOROOT" +fi +sudo mv "$NEW_GOROOT" /usr/local/go +sudo rm -rf "$OLD_GOROOT" +trap - ERR + sudo ln -sf /usr/local/go/bin/go /usr/bin/go sudo ln -sf /usr/local/go/bin/gofmt /usr/bin/gofmt + +# Remove the tarball +rm "$work_dir/$tarball" From e86893c2a58dfa72c06a3173daeee2fc7c5d2cfb Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Wed, 3 Jun 2026 14:25:37 +0530 Subject: [PATCH 66/95] Stop publishing Blobfuse2 to CBL-Mariner 2.0 (EOL) (#2239) --- CHANGELOG.md | 3 + blobfuse2-testartifacts.yaml | 1833 ---------------------------------- docker/dockerinstall.sh | 4 +- setup/packages.csv | 2 - 4 files changed, 5 insertions(+), 1837 deletions(-) delete mode 100644 blobfuse2-testartifacts.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index e88284004..161b51a6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ **Bug Fixes** - Fix CPK-encrypted blob attribute lookup duplicating the prefix path when subdirectory is configured ([PR #2199](https://github.com/Azure/azure-storage-fuse/pull/2199)) +**Other Changes** +- CBL-Mariner 2.0 has reached end-of-life; Blobfuse2 packages will no longer be published to Mariner 2.0 repositories. + ## 2.5.3 (2026-03-25) **Features** - Add rate limit functionality for ingress bandwidth (bytes downloaded per second) and operations per second ([PR #2093](https://github.com/Azure/azure-storage-fuse/pull/2093)) diff --git a/blobfuse2-testartifacts.yaml b/blobfuse2-testartifacts.yaml deleted file mode 100644 index 5103982ec..000000000 --- a/blobfuse2-testartifacts.yaml +++ /dev/null @@ -1,1833 +0,0 @@ -parameters: - - name: tag - displayName: 'New Release Tag' - type: string - default: 'blobfuse2-' - - - name: unit_test - displayName: 'Execute Unit Tests' - type: boolean - default: false - - - name: post_release - displayName: 'Post Release on Github' - type: boolean - default: false - - - name: publish_artifacts - displayName: 'Publish Artifacts to Linux Repos' - type: boolean - default: false - - - name: draft - displayName: 'Post as Draft Release' - type: boolean - default: false - - - name: prerelease - displayName: 'Post as PreRelease' - type: boolean - default: false - - - name: update_version - displayName: 'Update Version' - type: boolean - default: false - -# Do not trigger this pipeline automatically -trigger: none -pr: none - -stages: - - stage: BuildArtifacts - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Libfuse: - vmImage: 'ubuntu-18.04' - fuselib: 'libfuse-dev' - tags: 'fuse2' - depends: 'fuse' - container: 'test-cnt-ubn-18' - AgentName: "blobfuse-ubuntu18" - Libfuse3: - vmImage: 'ubuntu-20.04' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - depends: 'fuse3' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - # list commits from past 12hrs - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - git --no-pager log --since="12 hours ago" --stat - displayName: 'List Commits' - workingDirectory: $(work_dir) - - # install dependencies required for compiling blobfuse - - script: | - sudo apt-get update --fix-missing - sudo apt-get install ruby-dev build-essential pkg-config cmake gcc g++ rpm $(fuselib) -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -v 1.16 - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - # build blobfuse2 and generate binary - - template: 'azure-pipeline-templates/build-release.yml' - parameters: - work_dir: $(work_dir) - root_dir: $(root_dir) - unit_test: ${{ parameters.unit_test }} - tags: $(tags) - container: $(container) - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDir/usr/bin/ - mkdir -p pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2 - cp azure-storage-fuse/bfusemon pkgDir/usr/bin/bfusemon - cp azure-storage-fuse/setup/baseConfig.yaml pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/sampleFileCacheConfig.yaml pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/sampleStreamingConfig.yaml pkgDir/usr/share/blobfuse2/ - mkdir -p pkgDir/etc/rsyslog.d - mkdir -p pkgDir/etc/logrotate.d - cp azure-storage-fuse/setup/11-blobfuse2.conf pkgDir/etc/rsyslog.d - cp azure-storage-fuse/setup/blobfuse2-logrotate pkgDir/etc/logrotate.d/blobfuse2 - workingDirectory: $(root_dir) - displayName: 'Accumulate pkg files' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d $(depends) \ - --maintainer "Blobfuse v-Team " --url "https://github.com/Azure/azure-storage-fuse" \ - --description "An user-space filesystem for interacting with Azure Storage" - mv ./blobfuse2*.deb ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./blobfuse2*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make deb Package' - - - script: | - fpm -s dir -t rpm -n blobfuse2 --rpm-digest sha256 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d $(depends) \ - --maintainer "Blobfuse v-Team " --url "https://github.com/Azure/azure-storage-fuse" \ - --description "An user-space filesystem for interacting with Azure Storage" - mv ./blobfuse2*.rpm ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./blobfuse2*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make rpm Package' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2-temp' - displayName: 'Publish Artifacts' - - - job: Set_2_ARM - timeoutInMinutes: 120 - strategy: - matrix: - Libfuse3: - vmImage: 'ubuntu-22.04' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - depends: 'fuse3' - container: 'test-cnt-ubn-22-arm64' - AgentName: 'blobfuse-ubn22-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - # list commits from past 12hrs - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - git --no-pager log --since="12 hours ago" --stat - displayName: 'List Commits' - workingDirectory: $(work_dir) - - # install dependencies required for compiling blobfuse - - script: | - sudo apt-get update --fix-missing - sudo apt-get install ruby-dev build-essential pkg-config cmake gcc g++ rpm $(fuselib) -y - sudo gem install fpm -v 1.16 - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - # build blobfuse2 and generate binary - - template: 'azure-pipeline-templates/build-release.yml' - parameters: - work_dir: $(work_dir) - root_dir: $(root_dir) - unit_test: ${{ parameters.unit_test }} - tags: $(tags) - container: $(container) - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDir/usr/bin/ - mkdir -p pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2 - cp azure-storage-fuse/bfusemon pkgDir/usr/bin/bfusemon - cp azure-storage-fuse/setup/baseConfig.yaml pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/sampleFileCacheConfig.yaml pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/sampleStreamingConfig.yaml pkgDir/usr/share/blobfuse2/ - mkdir -p pkgDir/etc/rsyslog.d - mkdir -p pkgDir/etc/logrotate.d - cp azure-storage-fuse/setup/11-blobfuse2.conf pkgDir/etc/rsyslog.d - cp azure-storage-fuse/setup/blobfuse2-logrotate pkgDir/etc/logrotate.d/blobfuse2 - workingDirectory: $(root_dir) - displayName: 'Accumulate pkg files' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d $(depends) \ - --maintainer "Blobfuse v-Team " --url "https://github.com/Azure/azure-storage-fuse" \ - --description "An user-space filesystem for interacting with Azure Storage" - mv ./blobfuse2*.deb ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(tags).arm64.deb - cp ./blobfuse2*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make deb Package' - - - script: | - fpm -s dir -t rpm -n blobfuse2 --rpm-digest sha256 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d $(depends) \ - --maintainer "Blobfuse v-Team " --url "https://github.com/Azure/azure-storage-fuse" \ - --description "An user-space filesystem for interacting with Azure Storage" - mv ./blobfuse2*.rpm ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(tags).aarch64.rpm - cp ./blobfuse2*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make rpm Package' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2-temp' - displayName: 'Publish Artifacts' -# BuildArtifacts end here - - - stage: SignArtifacts - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') - jobs: - - job: SignBlobfuse - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb - md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm - displayName: 'List Artifacts' - - - script: | - mkdir mariner && chmod 755 mariner - cp blobfuse2-temp/*-fuse3*.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - script: | - sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" - - # Send images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning blobfuse2' - inputs: - ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/blobfuse2-temp' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_BLOBFUSE_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning blobfuse2 mariner' - inputs: - ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_BLOBFUSE_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm - chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/*.rpm - rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/blobfuse2-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/*.deb - md5sum $(Build.ArtifactStagingDirectory)/*.rpm - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2-signed' - displayName: 'Publish Signed Artifacts' - # SignArtifacts end here - - - stage: TestArtifacts - dependsOn: SignArtifacts - condition: succeeded('SignArtifacts') - jobs: - - job: Set_0 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - fuse-version: 'fuse3' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo dpkg --info blobfuse2*.deb - sudo dpkg -i blobfuse2*.deb - sudo apt-get install $(fuse-version) build-essential -y - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # Create container for blobfuse and publish it - - script: | - sudo apt-get install $(tags) $(fuselib) -y - chmod 777 *.sh - ./dockerinstall.sh - ./buildcontainer.sh Dockerfile x86_64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 - ./buildcontainer.sh DockerfileMariner mariner-x86_64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 - displayName: "Create and publish container" - workingDirectory: $(work_dir)/docker - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-18: - vmImage: 'Ubuntu-18.04' - fuselib: 'libfuse-dev' - fuse-version: 'fuse' - tags: 'fuse2' - container: 'test-cnt-ubn-18' - AgentName: "blobfuse-ubuntu18" - Ubuntu-20: - vmImage: 'Ubuntu-20.04' - fuse-version: 'fuse3' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - if [ $(AgentName) == "blobfuse-ubuntu18" ]; then - echo "Generating for Debian 9/10" - f=`ls ./blobfuse2*$(vmImage)*.deb` - cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") - cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") - cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) - ls -l $(Build.ArtifactStagingDirectory) - rm -rf ./blobfuse2*Debian-*.deb - else - echo "Generating for Debian 11" - f=`ls ./blobfuse2*$(vmImage)*.deb` - cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") - cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) - ls -l $(Build.ArtifactStagingDirectory) - rm -rf ./blobfuse2*Debian-*.deb - fi - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo dpkg --info blobfuse2*.deb - sudo apt-get install $(fuse-version) build-essential -y - sudo dpkg -i blobfuse2*.deb - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_2_ARM - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22-ARM64: - vmImage: 'Ubuntu-22.04' - fuse-version: 'fuse3' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*arm64.deb - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo dpkg --info blobfuse2*.deb - sudo apt-get install $(fuse-version) build-essential -y - sudo dpkg -i blobfuse2*.deb - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # Create container for blobfuse and publish it - - script: | - sudo apt-get install $(tags) $(fuselib) -y - chmod 777 *.sh - ./dockerinstall.sh - ./buildcontainer.sh DockerfileARM arm64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) arm64 - displayName: "Create and publish ARM container" - workingDirectory: $(work_dir)/docker - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_2_AARCH - timeoutInMinutes: 120 - strategy: - matrix: - RHEL-9.0: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - fuselib: 'fuse fuse3-libs fuse3-devel' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-rhel-9-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*aarch64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip blobfuse2*$(vmImage)*.rpm - sudo yum groupinstall "Development Tools" -y - sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing - sudo rpm -i blobfuse2*$(vmImage)*.rpm - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_4_1 - timeoutInMinutes: 120 - strategy: - matrix: - RHEL-7.5: - agentName: "blobfuse-rhel7_5" - vmImage: 'RHEL-7.5' - fuselib: 'fuse3-devel' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-rhel-75' - - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - sudo yum update -y - sudo yum install git -y - sudo yum groupinstall "Development Tools" -y - displayName: 'Install Git' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - f=`ls ./blobfuse2*$(vmImage)*.rpm` - cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") - cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") - cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") - cp ./blobfuse2*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) - cp ./blobfuse2*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) - rm -rf ./blobfuse2*RHEL-7.8*.rpm - rm -rf ./blobfuse2*RHEL-8*.rpm - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_4_2 - timeoutInMinutes: 120 - strategy: - matrix: - RHEL-8.6: - agentName: "blobfuse-rhel8_6" - vmImage: 'RHEL-8.6' - fuselib: 'fuse fuse3-libs fuse3-devel' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-rhel-86' - RHEL-9.0: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - fuselib: 'fuse fuse3-libs fuse3-devel' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-rhel-9' - - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip blobfuse2*$(vmImage)*.rpm - sudo yum groupinstall "Development Tools" -y - sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing - sudo rpm -i blobfuse2*$(vmImage)*.rpm - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - CentOS-7.9: - agentName: "blobfuse-centos7" - vmImage: 'CentOS-7.0' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-cent-7' - CentOS-8.5: - agentName: "blobfuse-centos8" - vmImage: 'CentOS-8.0' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-cent-8' - - pool: - name: "blobfuse-centos-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - condition: eq(variables['agentName'], 'blobfuse-centos8') - displayName: "Update OS mirrors" - - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo rpm -qip blobfuse2*.rpm - sudo yum install gcc gcc-c++ make -y - if [ $(agentName) == "blobfuse-centos8" ]; then - sudo yum install fuse fuse3 fuse3-devel -y --nobest --allowerasing - else - sudo yum install fuse fuse3 fuse3-devel -y - fi - sudo rpm -i blobfuse2*.rpm - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_6 - timeoutInMinutes: 120 - strategy: - matrix: - Oracle-8.1: - agentName: "blobfuse-oracle81" - vmImage: 'Oracle-8.1' - fuselib: 'fuse3 fuse3-devel' - fuse-version: 'fuse3' - tags: 'fuse3' - container: "test-cnt-ora-81" - - pool: - name: "blobfuse-oracle-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo rpm -qip blobfuse2*.rpm - sudo yum install gcc gcc-c++ make -y - sudo yum install fuse $(fuse-version) -y --nobest --allowerasing - sudo rpm -i blobfuse2*.rpm - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_7 - timeoutInMinutes: 120 - strategy: - matrix: - SUSE-15: - agentName: "blobfuse-suse15" - vmImage: 'SUSE-15Gen2' - fuse-version: 'fuse3' - tags: 'fuse3' - container: "test-cnt-suse-15" - - pool: - name: "blobfuse-suse-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - sudo zypper update -y - sudo zypper -n install git - displayName: 'Install Git' - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo rpm -qip blobfuse2*.rpm - sudo zypper -n install make cmake gcc gcc-c++ fuse fuse3 - wget https://rpmfind.net/linux/opensuse/distribution/leap/15.2/repo/oss/x86_64/fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm - sudo zypper -n --no-gpg-checks install fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm - sudo rm fuse3-devel*.rpm - sudo rpm -i blobfuse2*.rpm - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - - job: Set_8 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - fuselib: 'libfuse3-dev' - fuse-version: 'fuse3' - tags: 'fuse3' - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install Git' - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/mariner/blobfuse2*$(tags)*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed/mariner - - - script: | - sudo rpm -qip blobfuse2*.rpm - sudo tdnf install gcc build-essential fuse3 fuse3-devel -y - sudo rpm -i blobfuse2*.rpm - displayName: 'Install Package' - workingDirectory: $(root_dir)/blobfuse2-signed/mariner - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - - job: Set_9 - timeoutInMinutes: 120 - strategy: - matrix: - Rocky-8.0: - agentName: "blobfuse-rocky8" - vmImage: 'Rocky-8.0' - fuselib: 'fuse3-devel' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-rocky-8' - Rocky-9.0: - agentName: "blobfuse-rocky9" - vmImage: 'Rocky-9.0' - fuselib: 'fuse3-devel' - fuse-version: 'fuse3' - tags: 'fuse3' - container: 'test-cnt-rocky-9' - - pool: - name: "blobfuse2-rocky-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - steps: - - checkout: none - - - script: | - sudo yum update -y - sudo yum install wget git -y - displayName: 'Install Git' - - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip blobfuse2*$(vmImage)*.rpm - sudo yum groupinstall "Development Tools" -y - sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing - sudo rpm -i blobfuse2*$(vmImage)*.rpm - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - # TestArtifacts ends here - - stage: ReleaseArtifacts - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') - jobs: - - job: ReleaseBlobfuse - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.deb - md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.rpm - displayName: 'List Artifacts' - - - script: | - sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" - - - ${{ if eq(parameters.post_release, true) }}: - # add release tags & push to github - - task: GithubRelease@1 - inputs: - githubConnection: 'Azure' - repositoryName: 'Azure/azure-storage-fuse' - action: 'edit' - - target: '$(Build.SourceVersion)' - tagSource: 'userSpecifiedTag' - - title: ${{ parameters.tag }} - tag: ${{ parameters.tag }} - - assets: | - $(Build.ArtifactStagingDirectory)/blobfuse2/* - changeLogCompareToRelease: 'lastFullRelease' - changeLogType: 'commitBased' - isDraft: ${{ parameters.draft }} - isPreRelease: ${{ parameters.prerelease }} - assetUploadMode: replace - - - ${{ if eq(parameters.publish_artifacts, true) }}: - - stage: PublishArtifacts - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: PublishArtifacts - timeoutInMinutes: 120 - pool: - vmImage: 'ubuntu-22.04' - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - checkout: none - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'BlobFuse/Blobfuse' - displayName: 'Connect to PMC artifact' - - - script: | - pip install pmc-cli - displayName: 'Install pmc-cli' - - - task: DownloadSecureFile@1 - name: pmcCertificate - displayName: 'Download pmc pem file' - inputs: - secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - - task: DownloadSecureFile@1 - name: settings - displayName: 'Download settings.toml file' - inputs: - secureFile: 'settings.toml' - - - script: | - pmc --version - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Test PMC installation' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Signed Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - cd mariner - for f in ./blobfuse2*fuse3*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done - ls -lRt - mv blobfuse2*.rpm .. - cd .. - rm -r mariner/ - displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - displayName: 'List Artifacts' - - - script: | - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload blobfuse2-signed - displayName: 'Upload packages' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-fuse/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv - cat ./packages.csv - displayName: 'Fetch packages.csv' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` - echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" - - fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` - echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" - - fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` - echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" - - fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` - echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" - - fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` - echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" - - fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` - echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" - - marinerAmdRpmFile=$(ls blobfuse2* | grep 'cm2\.x86_64\.rpm') - marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` - echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" - echo "Mariner fuse3 AMD RPM $marinerAmdRpmFile" - - marinerAarchRpmFile=$(ls blobfuse2* | grep 'cm2\.aarch64\.rpm') - marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` - echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" - echo "Mariner fuse3 ARM RPM: $marinerAarchRpmFile" - - is_preview="false" - echo "##vso[task.setvariable variable=is_preview]$is_preview" - if [[ $marinerAmdRpmFile == *"preview"* ]]; then - is_preview="true" - echo "##vso[task.setvariable variable=is_preview]$is_preview" - fi - - is_preview="false" - echo "##vso[task.setvariable variable=is_preview]$is_preview" - if [[ $marinerFuse3AmdRpm == *"preview"* ]]; then - is_preview="true" - echo "##vso[task.setvariable variable=is_preview]$is_preview" - fi - - while IFS=, read -r distro fuseArchType repoName releaseName; do - - # If the package is preview, publish to mariner preview package - if [[ $distro == *"Mariner-"* ]]; then - if [ $is_preview = "true" ]; then - repoName=$(echo $repoName | sed 's/prod/preview/') - fi - fi - - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +3 ../packages.csv) - displayName: 'Add uploaded packages to repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - - script: | - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - if [[ $distro == *"Mariner-"* ]]; then - if [ "$(is_preview)" = "true" ]; then - repoName=$(echo $repoName | sed 's/prod/preview/') - fi - fi - echo "Repository Name: $repoName" - - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +3 ../packages.csv) - displayName: 'Publish the repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - - stage: UpdateLatestVersion - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: UpdateVersion - timeoutInMinutes: 120 - strategy: - matrix: - UBN20: - vmImage: 'ubuntu-20.04' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - depends: 'fuse3' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - checkout: none - - - ${{ if eq(parameters.update_version, true) }}: - - script: | - sudo apt-get install python3 -y - python3 --version - displayName: 'Installing Python' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2' - downloadPath: $(Build.ArtifactStagingDirectory) - - # install blobfuse2 - - script: | - sudo apt install fuse3 -y - cd $(Build.ArtifactStagingDirectory)/blobfuse2 - ls | grep -i ubuntu-20.04 - sudo apt-get install ./`ls | grep -i ubuntu-20.04` -y - blobfuse2 version - displayName: 'Installing blobfuse2' - - - script: | - mkdir -p ~/blob_mnt - chmod 777 ~/blob_mnt - mkdir -p ~/blobfusetemp - chmod 777 ~/blobfusetemp - blobfuse2 mount ~/blob_mnt --tmp-path=~/blobfusetemp --container-name="\$web" - sleep 2 - env: - AZURE_STORAGE_ACCOUNT: "blobfuse2" - AZURE_STORAGE_AUTH_TYPE: "msi" - AZURE_STORAGE_IDENTITY_CLIENT_ID: $(BLOBFUSE2_MSI_CIENT_ID) - displayName: 'Mount release container for version update' - - - script: | - cd ~/blob_mnt/release/latest/ - echo "Before cleanup" - ls -l - rm -rf * - echo "After cleanup" - ls -l - displayName: 'Delete older version file' - - - script: | - cd ~/blob_mnt/release/latest/ - touch `blobfuse2 version | cut -d " " -f3` - ls -l - displayName: 'Create new version file' - - - script: | - cd ~/blob_mnt/release/latest/ - version=`blobfuse2 version | cut -d " " -f3` - echo "Release version: " $version - echo -e "\n$version\n" > index.xml - echo "Printing index.xml" - cat index.xml - displayName: 'Update the XML file' - - - script: | - blobfuse2 unmount all - displayName: 'Unmount Blobfuse2' - diff --git a/docker/dockerinstall.sh b/docker/dockerinstall.sh index 32a977981..99055b149 100755 --- a/docker/dockerinstall.sh +++ b/docker/dockerinstall.sh @@ -1,8 +1,8 @@ # Cleanup old installation sudo apt remove docker-desktop -rm-r $HOME/.docker/desktop -sudo rm/usr/local/bin/com.docker.cli +rm -rf "$HOME/.docker/desktop" +sudo rm -f /usr/local/bin/com.docker.cli sudo apt purge docker-desktop sudo apt-get update diff --git a/setup/packages.csv b/setup/packages.csv index 6db97d228..f7498c2b9 100644 --- a/setup/packages.csv +++ b/setup/packages.csv @@ -24,8 +24,6 @@ RHEL-10.0,fuse3ArmRpm,microsoft-rhel10.0-prod-yum, CentOS-7.0,fuse3AmdRpm,microsoft-centos7-prod-yum, CentOS-8.0,fuse3AmdRpm,microsoft-centos8-prod-yum, SUSE-15Gen2,fuse3AmdRpm,microsoft-sles15-prod-yum, -Mariner-2.0-x86_64,marinerFuse3AmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, -Mariner-2.0-aarch64,marinerFuse3AarchRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, Rocky-8.0,fuse3AmdRpm,microsoft-el8-prod-yum, Rocky-9.0,fuse3AmdRpm,microsoft-el9-prod-yum, Mariner-3.0-x86_64,marinerFuse3AmdRpm,azurelinux-3.0-prod-ms-oss-x86_64-yum, From 81ec8b3c55202ac37e9b39415c9cfe83088b70fc Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Wed, 3 Jun 2026 15:50:06 +0530 Subject: [PATCH 67/95] Return correct posix error ENAMETOOLONG for dir creation in ADLS (#2221) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + component/azstorage/datalake.go | 3 +++ component/azstorage/utils.go | 3 +++ component/azstorage/utils_test.go | 33 +++++++++++++++++++++++++++ component/libfuse/libfuse2_handler.go | 2 ++ component/libfuse/libfuse_handler.go | 2 ++ test/e2e_tests/dir_test.go | 25 ++++++++++++++++++++ 7 files changed, 69 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 161b51a6a..1e699460f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ **Bug Fixes** - Fix CPK-encrypted blob attribute lookup duplicating the prefix path when subdirectory is configured ([PR #2199](https://github.com/Azure/azure-storage-fuse/pull/2199)) +- Return `ENAMETOOLONG` instead of `EIO` when creating a directory path that exceeds the ADLS depth limit of 63 segments ([PR #2221](https://github.com/Azure/azure-storage-fuse/pull/2221)) **Other Changes** - CBL-Mariner 2.0 has reached end-of-life; Blobfuse2 packages will no longer be published to Mariner 2.0 repositories. diff --git a/component/azstorage/datalake.go b/component/azstorage/datalake.go index cf5188eaf..df51b425a 100644 --- a/component/azstorage/datalake.go +++ b/component/azstorage/datalake.go @@ -278,6 +278,9 @@ func (dl *Datalake) CreateDirectory(name string) error { case ErrFileAlreadyExists: log.Err("Datalake::CreateDirectory : Path already exists for %s [%s]", name, err.Error()) return syscall.EEXIST + case ErrPathTooDeep: + log.Err("Datalake::CreateDirectory : Path is too deep for %s [%s]", name, err.Error()) + return syscall.ENAMETOOLONG default: log.Err("Datalake::CreateDirectory : Failed to create directory %s [%s]", name, err.Error()) return err diff --git a/component/azstorage/utils.go b/component/azstorage/utils.go index 31479cdcc..c7f67a742 100644 --- a/component/azstorage/utils.go +++ b/component/azstorage/utils.go @@ -240,6 +240,7 @@ const ( InvalidRange BlobIsUnderLease InvalidPermission + ErrPathTooDeep ) // For detailed error list refer below link, @@ -285,6 +286,8 @@ func storeDatalakeErrToErr(err error) uint16 { return BlobIsUnderLease case datalakeerror.AuthorizationPermissionMismatch: return InvalidPermission + case datalakeerror.PathIsTooDeep: + return ErrPathTooDeep default: return ErrUnknown } diff --git a/component/azstorage/utils_test.go b/component/azstorage/utils_test.go index 8db795b5b..e7e8865cf 100644 --- a/component/azstorage/utils_test.go +++ b/component/azstorage/utils_test.go @@ -42,6 +42,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/datalakeerror" "github.com/Azure/azure-storage-fuse/v2/common" "github.com/Azure/azure-storage-fuse/v2/common/log" "github.com/stretchr/testify/assert" @@ -477,6 +478,38 @@ func (s *utilsTestSuite) TestRemoveLeadingSlashes() { } } +func (s *utilsTestSuite) TestStoreDatalakeErrToErr() { + assert := assert.New(s.T()) + + type testCase struct { + name string + code datalakeerror.StorageErrorCode + expected uint16 + } + + inputs := []testCase{ + {name: "PathAlreadyExists", code: datalakeerror.PathAlreadyExists, expected: ErrFileAlreadyExists}, + {name: "PathNotFound", code: datalakeerror.PathNotFound, expected: ErrFileNotFound}, + {name: "SourcePathNotFound", code: datalakeerror.SourcePathNotFound, expected: ErrFileNotFound}, + {name: "LeaseIDMissing", code: datalakeerror.LeaseIDMissing, expected: BlobIsUnderLease}, + {name: "AuthorizationPermissionMismatch", code: datalakeerror.AuthorizationPermissionMismatch, expected: InvalidPermission}, + {name: "PathIsTooDeep", code: datalakeerror.PathIsTooDeep, expected: ErrPathTooDeep}, + {name: "Unknown", code: datalakeerror.StorageErrorCode("UnknownCode"), expected: ErrUnknown}, + } + + for _, i := range inputs { + s.Run(i.name, func() { + respErr := &azcore.ResponseError{ErrorCode: string(i.code)} + result := storeDatalakeErrToErr(respErr) + assert.Equal(i.expected, result) + }) + } + + // nil error returns ErrNoErr + result := storeDatalakeErrToErr(nil) + assert.Equal(uint16(ErrNoErr), result) +} + func (s *utilsTestSuite) TestRemovePrefixPath() { assert := assert.New(s.T()) diff --git a/component/libfuse/libfuse2_handler.go b/component/libfuse/libfuse2_handler.go index c7d2b6f64..2c2ac1fe7 100644 --- a/component/libfuse/libfuse2_handler.go +++ b/component/libfuse/libfuse2_handler.go @@ -441,6 +441,8 @@ func libfuse_mkdir(path *C.char, mode C.mode_t) C.int { return -C.EACCES } else if os.IsExist(err) { return -C.EEXIST + } else if errors.Is(err, syscall.ENAMETOOLONG) { + return -C.ENAMETOOLONG } else { return -C.EIO } diff --git a/component/libfuse/libfuse_handler.go b/component/libfuse/libfuse_handler.go index f22f69243..88edce4e2 100644 --- a/component/libfuse/libfuse_handler.go +++ b/component/libfuse/libfuse_handler.go @@ -445,6 +445,8 @@ func libfuse_mkdir(path *C.char, mode C.mode_t) C.int { return -C.EACCES } else if os.IsExist(err) { return -C.EEXIST + } else if errors.Is(err, syscall.ENAMETOOLONG) { + return -C.ENAMETOOLONG } else { return -C.EIO } diff --git a/test/e2e_tests/dir_test.go b/test/e2e_tests/dir_test.go index 000f7fe15..290b8bef5 100644 --- a/test/e2e_tests/dir_test.go +++ b/test/e2e_tests/dir_test.go @@ -45,6 +45,7 @@ import ( "path/filepath" "strconv" "strings" + "syscall" "testing" "time" @@ -255,6 +256,30 @@ func (suite *dirTestSuite) TestDirDeleteNonEmpty() { // suite.dirTestCleanup([]string{dirName}) // } +// # Create directory path exceeding ADLS depth limit +func (suite *dirTestSuite) TestDirCreateDeepPath() { + if !suite.adlsTest { + fmt.Println("Skipping TestDirCreateDeepPath: not an ADLS account") + return + } + + // ADLS enforces a maximum path depth of 63 segments from the container root. + // Build 65 nested levels from testPath to guarantee the limit is exceeded. + const depth = 65 + topDir := suite.testPath + "/deep" + deepPath := topDir + for i := range depth { + deepPath = filepath.Join(deepPath, fmt.Sprintf("l%d", i)) + } + + err := os.MkdirAll(deepPath, 0777) + suite.Error(err) + suite.ErrorIs(err, syscall.ENAMETOOLONG) + + // cleanup any intermediate directories created before the limit was hit + suite.dirTestCleanup([]string{topDir}) +} + // # Get stats of a directory func (suite *dirTestSuite) TestDirGetStats() { dirName := suite.testPath + "/test3" From e3737e044c07afb242d80bd0d1d881ea04bf859c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jun 2026 11:13:42 +0530 Subject: [PATCH 68/95] Bump github.com/Azure/azure-sdk-for-go/sdk/azcore from 1.21.1 to 1.22.0 (#2243) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index c17f2afd1..8927cf036 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Azure/azure-storage-fuse/v2 go 1.26.3 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.5.0 @@ -49,7 +49,7 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.51.0 // indirect - golang.org/x/net v0.54.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 golang.org/x/text v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index 799128f0d..f3a899205 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= @@ -113,8 +113,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 3f3f0bfcc3df2cd515c9668a5d6927fc15e8fdfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:57:43 +0530 Subject: [PATCH 69/95] Bump golang.org/x/sys from 0.45.0 to 0.46.0 (#2246) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8927cf036..5a19d37c0 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.51.0 // indirect golang.org/x/net v0.55.0 // indirect - golang.org/x/sys v0.45.0 + golang.org/x/sys v0.46.0 golang.org/x/text v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index f3a899205..a775d3988 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From bb288d4754c51255199fa66153b87cc7f35fc68e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:16:40 +0530 Subject: [PATCH 70/95] Bump gopkg.in/ini.v1 from 1.67.2 to 1.67.3 (#2247) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: syeleti-msft --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5a19d37c0..2e5ada01e 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/vibhansa-msft/tlru v0.0.0-20240410102558-9e708419e21f go.uber.org/atomic v1.11.0 golang.org/x/time v0.15.0 - gopkg.in/ini.v1 v1.67.2 + gopkg.in/ini.v1 v1.67.3 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index a775d3988..b850fea99 100644 --- a/go.sum +++ b/go.sum @@ -141,8 +141,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= -gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From fefa01301680c3eba6216a4005684081cbba0ce6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:21:51 +0530 Subject: [PATCH 71/95] Bump github.com/Azure/azure-sdk-for-go/sdk/azidentity from 1.13.1 to 1.14.0 (#2253) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2e5ada01e..715f6c58e 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.3 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.5.0 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda diff --git a/go.sum b/go.sum index b850fea99..da4abb0af 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= From 5b3218f873873691e721e49dcc83f4363ad5de76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:53:01 +0530 Subject: [PATCH 72/95] Bump github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake from 1.5.0 to 1.6.0 (#2252) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 715f6c58e..eacb26e76 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.26.3 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.5.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.6.0 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda github.com/fsnotify/fsnotify v1.10.1 github.com/go-viper/mapstructure/v2 v2.5.0 @@ -48,8 +48,8 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect - golang.org/x/net v0.55.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/sys v0.46.0 - golang.org/x/text v0.37.0 // indirect + golang.org/x/text v0.38.0 // indirect ) diff --git a/go.sum b/go.sum index da4abb0af..615948670 100644 --- a/go.sum +++ b/go.sum @@ -8,10 +8,10 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 h1:BM85pSYlVYQHdq00nxyPoOkyLF5NArJG3bOsrmbwr4k= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0/go.mod h1:QYjP2cB7ZYtS/8jAbE0VSBZde/tjExqGjp+8JY6/+ts= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.5.0 h1:xQ+0Of0JPDx03Fgq64eblLeFhAKv3ZtxT5Eg5QxkZFU= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.5.0/go.mod h1:SSAgdUeg1SlBmOHGAsmLK5m14YvdjLoJB3tZi51zukU= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.6.0 h1:1NwcufKC1Bass7HB1TWKi5uFvJkHZx5dZPTWg9XN3RQ= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.6.0/go.mod h1:cxYrnYegMPOfdVvWjAaKmBNDdElwk09NFnmIQs+0V1E= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= @@ -107,14 +107,14 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -128,8 +128,8 @@ golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 605ae8144e7cc5175a59bd0037e1a3f847da8f16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 10:08:46 +0530 Subject: [PATCH 73/95] Bump actions/checkout from 6 to 7 (#2259) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/arm-ci.yml | 2 +- .github/workflows/benchmark.yml | 2 +- .github/workflows/blobfuse-buddy.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/codespell.yml | 2 +- .github/workflows/issueMetrics.yml | 4 ++-- .github/workflows/mirror-to-blob.yml | 2 +- .github/workflows/pr-mirror.yml | 2 +- .github/workflows/publish-container.yml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/update-latest-version.yml | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index efa17e8d9..1a4440cc4 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup apt (install prerequisites) run: | diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8adc9d0b7..057624237 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,7 +39,7 @@ jobs: steps: - name: 'Checkout Blobfuse2' - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: ref: ${{ github.ref }} diff --git a/.github/workflows/blobfuse-buddy.yml b/.github/workflows/blobfuse-buddy.yml index 5d53034a5..8b6301633 100644 --- a/.github/workflows/blobfuse-buddy.yml +++ b/.github/workflows/blobfuse-buddy.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Azure Login (OIDC) uses: azure/login@v3 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5cf2a5ab2..632345e6e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 68ffb5674..f8936494a 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -16,7 +16,7 @@ jobs: name: Check for spelling errors runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: codespell-project/actions-codespell@master with: check_filenames: true diff --git a/.github/workflows/issueMetrics.yml b/.github/workflows/issueMetrics.yml index 495ebe716..d21a9b5f5 100644 --- a/.github/workflows/issueMetrics.yml +++ b/.github/workflows/issueMetrics.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Python uses: actions/setup-python@v6 @@ -33,7 +33,7 @@ jobs: python scripts/issueMetrics.py --days 90 > issueMetric.txt - name: Checkout benchmarks branch - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: ref: benchmarks path: benchmarks-branch diff --git a/.github/workflows/mirror-to-blob.yml b/.github/workflows/mirror-to-blob.yml index f3811c634..6c28a8d7c 100644 --- a/.github/workflows/mirror-to-blob.yml +++ b/.github/workflows/mirror-to-blob.yml @@ -20,7 +20,7 @@ jobs: AZURE_STORAGE_ACCOUNT: blobfusemcpserver steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 # ✅ Azure authentication (OIDC) - name: Azure login diff --git a/.github/workflows/pr-mirror.yml b/.github/workflows/pr-mirror.yml index b47155b67..1499b36ff 100644 --- a/.github/workflows/pr-mirror.yml +++ b/.github/workflows/pr-mirror.yml @@ -75,7 +75,7 @@ jobs: exit 1 - name: Checkout repo (no fork code) - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 1 diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index 4dac80513..6d01f42d9 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 05f0748c7..15c07b622 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Build Blobfuse2 run: | diff --git a/.github/workflows/update-latest-version.yml b/.github/workflows/update-latest-version.yml index 7c9ee1a76..1313f0a6a 100644 --- a/.github/workflows/update-latest-version.yml +++ b/.github/workflows/update-latest-version.yml @@ -27,7 +27,7 @@ jobs: steps: # Checkout the benchmarks branch where release metadata lives. - name: Checkout benchmarks branch - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: ref: benchmarks fetch-depth: 0 From ea371284ca993ce34be9e8f32954a119337afee0 Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Tue, 23 Jun 2026 09:41:04 +0530 Subject: [PATCH 74/95] remove ai bot that is auto commenting in issues (#2256) --- .github/workflows/blobfuse-buddy.yml | 117 --------- .github/workflows/issueMetrics.yml | 62 ----- .github/workflows/mirror-to-blob.yml | 119 --------- scripts/askbuddy.py | 19 -- scripts/call_agent.py | 331 ------------------------ scripts/deepwiki_query.py | 71 ------ scripts/export_github_history.py | 262 ------------------- scripts/fetch_public_docs.py | 30 --- scripts/issueMetrics.py | 366 --------------------------- 9 files changed, 1377 deletions(-) delete mode 100644 .github/workflows/blobfuse-buddy.yml delete mode 100644 .github/workflows/issueMetrics.yml delete mode 100644 .github/workflows/mirror-to-blob.yml delete mode 100644 scripts/askbuddy.py delete mode 100644 scripts/call_agent.py delete mode 100644 scripts/deepwiki_query.py delete mode 100644 scripts/export_github_history.py delete mode 100644 scripts/fetch_public_docs.py delete mode 100644 scripts/issueMetrics.py diff --git a/.github/workflows/blobfuse-buddy.yml b/.github/workflows/blobfuse-buddy.yml deleted file mode 100644 index 8b6301633..000000000 --- a/.github/workflows/blobfuse-buddy.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Blobfuse Buddy – AI Auto Reply - -on: - workflow_dispatch: - inputs: - event_type: - description: "Mock event type for manual testing" - required: true - default: "issue" - type: choice - options: - - issue - - discussion - title: - description: "Mock title" - required: false - default: "Blobfuse config question: direct-io vs disable-kernel-cache" - type: string - body: - description: "Mock body" - required: false - default: "What is the difference between direct-io and disable-kernel-cache in Blobfuse2, and which option is generally preferred?" - type: string - issues: - types: [opened] - discussion: - types: [created] - -permissions: - id-token: write - contents: read - issues: write - discussions: write - -jobs: - buddy-reply: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Azure Login (OIDC) - uses: azure/login@v3 - with: - client-id: ${{ secrets.AZURE_MCP_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_MCP_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_MCP_SUBSCRIPTION_ID }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - pip install openai azure-identity requests mcp httpx python-dotenv - - - name: Create mock event payload (manual test) - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - GITHUB_EVENT_NAME: ${{ github.event_name }} - ORIGINAL_EVENT_PATH: ${{ github.event_path }} - REPOSITORY: ${{ github.repository }} - run: | - python - <<'PY' - import json - import os - - event_name = os.environ["GITHUB_EVENT_NAME"] - event_type = "issue" - title = "Blobfuse config question: direct-io vs disable-kernel-cache" - body = "What is the difference between direct-io and disable-kernel-cache in Blobfuse2, and which option is generally preferred?" - repository = os.environ["REPOSITORY"] - - if event_name == "workflow_dispatch": - with open(os.environ["ORIGINAL_EVENT_PATH"], "r", encoding="utf-8") as event_file: - original_event = json.load(event_file) - inputs = original_event.get("inputs", {}) - event_type = inputs.get("event_type", event_type) - title = inputs.get("title", title) - body = inputs.get("body", body) - - if event_type == "issue": - payload = { - "issue": { - "title": title, - "body": body, - "comments_url": f"https://api.github.com/repos/{repository}/issues/1/comments" - } - } - else: - payload = { - "discussion": { - "title": title, - "body": body, - "node_id": "MOCK_NODE_ID", - "comments_url": f"https://api.github.com/repos/{repository}/discussions/1/comments" - } - } - - with open("mock_event.json", "w", encoding="utf-8") as file: - json.dump(payload, file) - PY - - - name: Call Azure AI Foundry Agent - env: - GITHUB_EVENT_PATH: ${{ github.event_name == 'workflow_dispatch' && format('{0}/mock_event.json', github.workspace) || github.event_path }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && 'true' || 'false' }} - MOCK_TITLE: "Blobfuse config question: direct-io vs disable-kernel-cache" - MOCK_BODY: "What is the difference between direct-io and disable-kernel-cache in Blobfuse2, and which option is generally preferred?" - FOUNDRY_BASE_URL: https://blobfuse-buddy-resource.services.ai.azure.com/api/projects/blobfuse-buddy/applications/blobfuse-buddy/protocols/openai - FOUNDRY_API_VERSION: 2025-11-15-preview - DEEPWIKI_REPO: ${{ github.repository }} - run: | - python scripts/call_agent.py diff --git a/.github/workflows/issueMetrics.yml b/.github/workflows/issueMetrics.yml deleted file mode 100644 index d21a9b5f5..000000000 --- a/.github/workflows/issueMetrics.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Monthly issue metrics -on: - workflow_dispatch: - schedule: - - cron: "0 0 1 * *" - -permissions: - contents: write - -jobs: - build: - name: issue metrics - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install PyGithub - - - name: Generate 90-day issue and PR metrics report - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO_NAME: ${{ github.repository }} - run: | - python scripts/issueMetrics.py --days 90 > issueMetric.txt - - - name: Checkout benchmarks branch - uses: actions/checkout@v7 - with: - ref: benchmarks - path: benchmarks-branch - - - name: Publish report to benchmarks branch - shell: bash - run: | - set -e - report_date=$(date -u +%Y-%m-%d) - - cp issueMetric.txt benchmarks-branch/issueMetric.txt - mkdir -p benchmarks-branch/issueMetrics - cp issueMetric.txt benchmarks-branch/issueMetrics/issueMetric-${report_date}.txt - - cd benchmarks-branch - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add issueMetric.txt issueMetrics/ - - if git diff --cached --quiet; then - echo "No report changes to commit." - exit 0 - fi - - git commit -m "Publish issue metrics report for ${report_date}" - git push origin benchmarks \ No newline at end of file diff --git a/.github/workflows/mirror-to-blob.yml b/.github/workflows/mirror-to-blob.yml deleted file mode 100644 index 6c28a8d7c..000000000 --- a/.github/workflows/mirror-to-blob.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: Mirror GitHub Content to Blob Storage - -on: - workflow_dispatch: - schedule: - - cron: "0 0 * * 0" # every Sunday at 00:00 UTC - -permissions: - id-token: write - contents: read - issues: read - pull-requests: read - -jobs: - mirror: - runs-on: ubuntu-latest - timeout-minutes: 180 # ✅ 3 hours - - env: - AZURE_STORAGE_ACCOUNT: blobfusemcpserver - - steps: - - uses: actions/checkout@v7 - - # ✅ Azure authentication (OIDC) - - name: Azure login - uses: azure/login@v3 - with: - client-id: ${{ secrets.AZURE_MCP_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_MCP_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_MCP_SUBSCRIPTION_ID }} - - # ✅ Upload repo code - - name: Upload repo code - run: | - az storage copy \ - --auth-mode login \ - --account-name $AZURE_STORAGE_ACCOUNT \ - --source . \ - --destination "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/repo-code/azure-storage-fuse" \ - --recursive \ - --exclude-path ".git" - - # ✅ Upload wiki - - name: Clone wiki - run: | - git clone https://github.com/Azure/azure-storage-fuse.wiki.git wiki - - - name: Upload wiki docs - run: | - az storage blob upload-batch \ - --auth-mode login \ - --account-name $AZURE_STORAGE_ACCOUNT \ - --destination repo-docs \ - --source wiki \ - --destination-path wiki - - - # ✅ Upload public documentation - - name: Fetch public documentation - run: | - pip install html2text - python scripts/fetch_public_docs.py - - - name: Upload public documentation - run: | - az storage blob upload-batch \ - --auth-mode login \ - --account-name $AZURE_STORAGE_ACCOUNT \ - --container-name repo-docs \ - --source public \ - --destination public - - # ✅ Export issues / PRs - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip - python3 -m pip install requests - python3 -m pip install html2text - - # ✅ Restore incremental state and existing JSONL exports so the exporter - # can append new/updated records to the full historical files rather - # than to an empty workspace, preserving the complete history in blob. - - name: Restore incremental state - run: | - mkdir -p out_github_export - az storage blob download-batch \ - --auth-mode login \ - --account-name $AZURE_STORAGE_ACCOUNT \ - --source repo-gh \ - --destination out_github_export \ - --no-progress \ - || echo "No previous export in blob - first run will perform a full export" - - - name: Export GitHub history - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - python3 scripts/export_github_history.py - - # ✅ Upload GitHub history (including updated state.json for next incremental run). - # --overwrite ensures all exported JSONL files and state.json are replaced with the - # latest data from this run. - - name: Upload GitHub history - run: | - az storage blob upload-batch \ - --auth-mode login \ - --account-name $AZURE_STORAGE_ACCOUNT \ - --destination repo-gh \ - --source out_github_export \ - --overwrite - - - \ No newline at end of file diff --git a/scripts/askbuddy.py b/scripts/askbuddy.py deleted file mode 100644 index 932f91c55..000000000 --- a/scripts/askbuddy.py +++ /dev/null @@ -1,19 +0,0 @@ -from openai import OpenAI -from azure.identity import DefaultAzureCredential, get_bearer_token_provider - -token_provider = get_bearer_token_provider( - DefaultAzureCredential(), - "https://ai.azure.com/.default" -) - -client = OpenAI( - api_key=token_provider, - base_url="https://blobfuse-buddy-resource.services.ai.azure.com/api/projects/blobfuse-buddy/applications/blobfuse-buddy/protocols/openai", - default_query={"api-version": "2025-11-15-preview"} -) - -response = client.responses.create( - input="Test request: Should I use direct-io or disable-kernel-cache?" -) - -print(response.output_text) \ No newline at end of file diff --git a/scripts/call_agent.py b/scripts/call_agent.py deleted file mode 100644 index 7e415309d..000000000 --- a/scripts/call_agent.py +++ /dev/null @@ -1,331 +0,0 @@ -import json -import os -import re -import sys -import requests -from pathlib import Path -from openai import OpenAI -from azure.identity import DefaultAzureCredential, get_bearer_token_provider - -# ----------------------------- -# Load GitHub event payload -# ----------------------------- -with open(os.environ["GITHUB_EVENT_PATH"], "r") as f: - event = json.load(f) - -is_issue = "issue" in event -is_discussion = "discussion" in event - -if is_issue: - title = event["issue"]["title"] - body = event["issue"].get("body", "") - comments_url = event["issue"]["comments_url"] -elif is_discussion: - title = event["discussion"]["title"] - body = event["discussion"].get("body", "") - discussion_node_id = event["discussion"]["node_id"] - comments_url = None # discussions use GraphQL, not REST -else: - dry_run = os.getenv("DRY_RUN", "false").lower() == "true" - if not dry_run: - raise RuntimeError("Unsupported GitHub event") - title = os.getenv("MOCK_TITLE", "Dry-run workflow test") - body = os.getenv("MOCK_BODY", "No issue/discussion payload found; using dry-run fallback.") - repository = os.getenv("GITHUB_REPOSITORY", "owner/repo") - comments_url = f"https://api.github.com/repos/{repository}/issues/1/comments" - - -# ----------------------------- -# Query DeepWiki for context (via MCP SSE) -# ----------------------------- -DEEPWIKI_REPO = os.getenv("DEEPWIKI_REPO") # e.g. "Azure/azure-storage-fuse" - -def query_deepwiki(question, repo_name): - """Ask DeepWiki a question about the repo via MCP SSE and return its answer.""" - if not repo_name: - print("DEEPWIKI_REPO not set, skipping DeepWiki query.") - return "" - try: - import subprocess - result = subprocess.run( - [sys.executable, os.path.join(os.path.dirname(__file__), "deepwiki_query.py"), - repo_name, question, ""], - capture_output=True, text=True, timeout=120, - ) - if result.returncode != 0: - print(f"DeepWiki query failed (non-fatal): {result.stderr.strip()}") - return "" - answer = result.stdout.strip() - print(f"DeepWiki returned {len(answer)} chars.") - print("----- DeepWiki Answer -----") - print(answer) - print("----- End DeepWiki Answer -----") - return answer - except Exception as e: - print(f"DeepWiki query failed (non-fatal): {e}") - return "" - - -# ----------------------------- -# Search local repo docs for context -# ----------------------------- -DOC_FILES = [ - "TSG.md", - "README.md", - "MIGRATION.md", - "KnownLimitations.txt", - "CHANGELOG.md", - "setup/baseConfig.yaml", - "setup/advancedConfig.yaml", - "setup/readme.md", - "sampleFileCacheConfig.yaml", - "sampleBlockCacheConfig.yaml", - "sampleFileCacheWithSASConfig.yaml", -] -# Include all CLI docs from doc/ -DOC_DIR = Path("doc") -if DOC_DIR.is_dir(): - DOC_FILES.extend(str(p) for p in DOC_DIR.glob("*.md")) - -MAX_CONTEXT_CHARS = 12000 # cap to avoid overwhelming the prompt - - -def search_local_docs(question): - """Search repo documentation for paragraphs relevant to the question.""" - # Extract keywords (3+ char words, lowercased, deduplicated) - words = set(re.findall(r"[a-z][a-z0-9_-]{2,}", question.lower())) - # Add common blobfuse terms that might appear as sub-strings - extra = {"mount", "unmount", "cache", "fuse", "config", "auth", "sas", - "spn", "msi", "key", "permission", "error", "fail", "timeout"} - keywords = words | (extra & set(question.lower().split())) - - scored_chunks = [] - for filepath in DOC_FILES: - path = Path(filepath) - if not path.is_file(): - continue - try: - text = path.read_text(encoding="utf-8", errors="replace") - except OSError: - continue - - # Split into paragraphs (double-newline or markdown heading boundary) - paragraphs = re.split(r"\n{2,}|\n(?=#)", text) - for para in paragraphs: - para = para.strip() - if len(para) < 30: - continue - lower_para = para.lower() - score = sum(1 for kw in keywords if kw in lower_para) - if score > 0: - scored_chunks.append((score, filepath, para)) - - # Sort by relevance and assemble context within budget - scored_chunks.sort(key=lambda x: x[0], reverse=True) - context_parts = [] - total = 0 - for score, source, chunk in scored_chunks: - if total + len(chunk) > MAX_CONTEXT_CHARS: - break - context_parts.append(f"[Source: {source}]\n{chunk}") - total += len(chunk) - - result = "\n\n---\n\n".join(context_parts) - print(f"Local doc search: {len(context_parts)} relevant snippets, {total} chars.") - return result - - -# ----------------------------- -# Gather context -# ----------------------------- -user_question = f"{title}\n{body}" -deepwiki_context = query_deepwiki(user_question, DEEPWIKI_REPO) -local_doc_context = search_local_docs(user_question) - -# ----------------------------- -# Build prompt -# ----------------------------- -context_block = "" -if deepwiki_context: - context_block += f""" ---- DeepWiki Reference Answer --- -{deepwiki_context} ---- End DeepWiki Reference --- -""" -if local_doc_context: - context_block += f""" ---- Relevant Documentation Snippets --- -{local_doc_context} ---- End Documentation --- -""" - -prompt = f""" -New GitHub item received. - -Title: -{title} - -Description: -{body} -{context_block} -Using the reference information above as grounding context, please analyze the issue/question -and provide a helpful, accurate response. Prioritize factual accuracy from the documentation. -Where the documentation doesn't cover the topic, use your own knowledge but note it clearly. -""" - -# ----------------------------- -# Call Azure AI Foundry Agent -# ----------------------------- -dry_run = os.getenv("DRY_RUN", "false").lower() == "true" -token_provider = get_bearer_token_provider( - DefaultAzureCredential(), - "https://ai.azure.com/.default" -) - -client = OpenAI( - api_key=token_provider, - base_url=os.environ["FOUNDRY_BASE_URL"], - default_query={"api-version": os.environ["FOUNDRY_API_VERSION"]} -) - -response = client.responses.create( - input=prompt, - timeout=60 -) - -agent_reply = response.output_text.strip() - -# ----------------------------- -# Strip unwanted sections from agent reply -# ----------------------------- -# Remove "Summary Table" and "Primary Sources" sections (and variants) -agent_reply = re.sub( - r"(?:^|\n)#+\s*\*?\*?Summary\s+Table\*?\*?.*?(?=\n#+\s|\Z)", - "", agent_reply, flags=re.DOTALL | re.IGNORECASE -) -agent_reply = re.sub( - r"(?:^|\n)\*?\*?Summary\s+Table\*?\*?\s*\n\|.*?(?=\n#+\s|\n[^|\s]|\Z)", - "", agent_reply, flags=re.DOTALL | re.IGNORECASE -) -agent_reply = re.sub( - r"(?:^|\n)#+\s*\*?\*?(?:Primary\s+Sources?|References?)\*?\*?.*?(?=\n#+\s|\Z)", - "", agent_reply, flags=re.DOTALL | re.IGNORECASE -) -agent_reply = re.sub( - r"(?:^|\n)\*?\*?(?:Primary\s+Sources?|References?)[:\*]*\*?\*?\s*\n.*?(?=\n#+\s|\n\n[^-*\s]|\Z)", - "", agent_reply, flags=re.DOTALL | re.IGNORECASE -) -agent_reply = agent_reply.strip() - -# ----------------------------- -# Prepend AI disclaimer -# ----------------------------- -final_reply = f"""⚠️ **AI‑generated response** - -This response was generated by an AI agent and may not be fully accurate. -Please validate recommendations before acting on them. - ---- - -{agent_reply} -""" - -# ----------------------------- -# Auto-summarize if too long -# ----------------------------- -GITHUB_COMMENT_CHAR_LIMIT = 65536 -SAFE_LIMIT = 60000 # leave headroom for markdown overhead -SUMMARIZE_TARGET = 55000 # target length for the summarized version - -if len(final_reply) > SAFE_LIMIT: - print(f"Response length ({len(final_reply)} chars) exceeds safe limit ({SAFE_LIMIT}). Requesting summarization...") - summarize_prompt = ( - f"The following answer is too long to post as a single GitHub comment " - f"(limit ~{GITHUB_COMMENT_CHAR_LIMIT} characters). " - f"Please condense it to at most {SUMMARIZE_TARGET} characters while preserving " - f"all key information, code examples, and actionable recommendations. " - f"Keep the same markdown formatting.\n\n" - f"--- ORIGINAL ANSWER ---\n{agent_reply}" - ) - try: - summary_resp = client.responses.create( - input=summarize_prompt, - timeout=90, - ) - summarized = summary_resp.output_text.strip() - final_reply = f"""⚠️ **AI‑generated response** *(summarized — original exceeded comment size limit)* - -This response was generated by an AI agent and may not be fully accurate. -Please validate recommendations before acting on them. - ---- - -{summarized} -""" - print(f"Summarized response to {len(final_reply)} chars.") - except Exception as e: - print(f"Summarization failed ({e}). Falling back to hard truncation.") - -# Hard-truncate as a last-resort safety net -if len(final_reply) > GITHUB_COMMENT_CHAR_LIMIT: - truncation_notice = "\n\n---\n⚠️ *Response was truncated because it exceeded the GitHub comment size limit.*" - max_body = GITHUB_COMMENT_CHAR_LIMIT - len(truncation_notice) - final_reply = final_reply[:max_body] + truncation_notice - print(f"Hard-truncated response to {len(final_reply)} chars.") - -if dry_run: - question = f"Title: {title}\nDescription: {body}" - print("DRY_RUN enabled. Skipping GitHub comment post.") - print(f"Target URL: {comments_url}") - print("----- Validation Q&A -----") - print("Question Asked:") - print(question) - print() - print("Agent Answer:") - print(final_reply) - print("----- End Validation Q&A -----") - raise SystemExit(0) - -# ----------------------------- -# Post reply back to GitHub -# ----------------------------- -headers = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github+json", -} - -try: - if is_discussion: - # GitHub discussions require the GraphQL API for commenting - graphql_url = "https://api.github.com/graphql" - mutation = """ - mutation($discussionId: ID!, $body: String!) { - addDiscussionComment(input: {discussionId: $discussionId, body: $body}) { - comment { id url } - } - } - """ - gql_resp = requests.post( - graphql_url, - headers=headers, - json={"query": mutation, "variables": {"discussionId": discussion_node_id, "body": final_reply}}, - ) - gql_resp.raise_for_status() - gql_data = gql_resp.json() - if "errors" in gql_data: - print(f"GraphQL errors: {gql_data['errors']}", file=sys.stderr) - sys.exit(1) - print(f"Discussion comment posted: {gql_data['data']['addDiscussionComment']['comment']['url']}") - else: - # Issues use the REST API - post_resp = requests.post( - comments_url, - headers=headers, - json={"body": final_reply}, - ) - post_resp.raise_for_status() - print(f"Issue comment posted: {post_resp.json().get('html_url', comments_url)}") -except requests.exceptions.HTTPError as e: - print(f"Failed to post comment: {e}", file=sys.stderr) - print(f"Response body: {e.response.text}", file=sys.stderr) - sys.exit(1) \ No newline at end of file diff --git a/scripts/deepwiki_query.py b/scripts/deepwiki_query.py deleted file mode 100644 index 9ab014b69..000000000 --- a/scripts/deepwiki_query.py +++ /dev/null @@ -1,71 +0,0 @@ -import asyncio -import os -import sys -from contextlib import AsyncExitStack -from typing import Optional -import subprocess - -import httpx -from mcp import ClientSession -from mcp.client.streamable_http import streamablehttp_client -from dotenv import load_dotenv - -load_dotenv() - -# The URL for the DeepWiki MCP server's Streamable HTTP transport -MCP_URL = "https://mcp.deepwiki.com/mcp" - -class MCPClient: - def __init__(self): - self.session: Optional[ClientSession] = None - self._streams_context = None - self._session_context = None - - async def connect_to_server(self, server_url: str): - self._streams_context = streamablehttp_client(url=server_url) - streams = await self._streams_context.__aenter__() - # streamablehttp_client yields (read, write, get_session_id) - self._session_context = ClientSession(streams[0], streams[1]) - self.session = await self._session_context.__aenter__() - await self.session.initialize() - - async def cleanup(self): - if self._session_context: - await self._session_context.__aexit__(None, None, None) - if self._streams_context: - await self._streams_context.__aexit__(None, None, None) - - async def ask_deepwiki(self, repo: str, question: str) -> str: - if not self.session: - raise RuntimeError("Client not connected.") - - # The MCP SDK handles the JSON-RPC call for you - result = await self.session.call_tool( - "ask_question", {"repoName": repo, "question": question} - ) - - # Join the list of content parts into a single string - return result.content - -async def main(repo, title, body): - client = MCPClient() - try: - await client.connect_to_server(server_url=MCP_URL) - - question = f"{title}\n\n{body}" - response = await client.ask_deepwiki(repo, question) - print (response) - - finally: - await client.cleanup() - -if __name__ == "__main__": - if len(sys.argv) < 4: - print("Usage: python deepwiki_query.py ") - sys.exit(1) - - repo_arg = sys.argv[1] - issue_title = sys.argv[2] - issue_body = sys.argv[3] - - asyncio.run(main(repo_arg, issue_title, issue_body)) diff --git a/scripts/export_github_history.py b/scripts/export_github_history.py deleted file mode 100644 index c0d280e57..000000000 --- a/scripts/export_github_history.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python3 -""" -Export GitHub repo issues + PRs + comments into JSONL files suitable for Azure AI Search blob indexer -using parsingMode: jsonLines (each line = one JSON document). - -Repo targeted: Azure/azure-storage-fuse (can be overridden by env vars OWNER/REPO). - -Outputs (in OUT_DIR, default: out_github_export): - - threads_issues_prs.jsonl (one doc per issue or PR thread) - - issue_pr_comments.jsonl (one doc per issue/PR timeline comment) - - pr_reviews.jsonl (one doc per PR review) - - pr_review_comments.jsonl (one doc per PR diff review comment) - -Incremental: - Stores latest updated_at in out_github_export/state.json and uses it as "since" next run. -""" - -import os -import sys -import json -import time -import hashlib -from urllib.parse import urlencode - -import requests - -OWNER = os.getenv("OWNER", "Azure") -REPO = os.getenv("REPO", "azure-storage-fuse") -BASE = "https://api.github.com" - -# Optional: set GITHUB_TOKEN to increase rate limits -GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "").strip() - -OUT_DIR = os.getenv("OUT_DIR", "out_github_export") -STATE_FILE = os.path.join(OUT_DIR, "state.json") - - -def _headers(): - h = { - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "blobfuse-support-exporter", - } - if GITHUB_TOKEN: - h["Authorization"] = f"Bearer {GITHUB_TOKEN}" - return h - - -def _ensure_out_dir(): - os.makedirs(OUT_DIR, exist_ok=True) - - -def _load_state(): - if os.path.exists(STATE_FILE): - with open(STATE_FILE, "r", encoding="utf-8") as f: - return json.load(f) - return {"since": None} - - -def _save_state(state): - with open(STATE_FILE, "w", encoding="utf-8") as f: - json.dump(state, f, indent=2) - - -def _request_json(url, params=None, max_retries=3): - if params: - url = f"{url}?{urlencode(params)}" - - retries = 0 - while True: - r = requests.get(url, headers=_headers(), timeout=60) - - # Basic rate limit handling - if r.status_code == 403 and "rate limit" in r.text.lower(): - reset = r.headers.get("X-RateLimit-Reset") - if reset: - sleep_for = max(1, int(reset) - int(time.time()) + 5) - print(f"[rate-limit] sleeping {sleep_for}s", file=sys.stderr) - time.sleep(sleep_for) - continue - - # Retry on transient 5xx errors - if r.status_code >= 500 and retries < max_retries: - retries += 1 - sleep_for = 2 ** retries # exponential backoff: 2, 4, 8 seconds - print(f"[retry {retries}/{max_retries}] {r.status_code} - sleeping {sleep_for}s", file=sys.stderr) - time.sleep(sleep_for) - continue - - r.raise_for_status() - return r.json(), r.headers - - -def _paginate(url, params=None): - params = params or {} - page = 1 - while True: - p = dict(params) - p["per_page"] = 100 - p["page"] = page - data, _hdrs = _request_json(url, p) - if not data: - break - for item in data: - yield item - page += 1 - - -def _stable_id(*parts): - raw = "|".join(str(p) for p in parts) - return hashlib.sha256(raw.encode("utf-8")).hexdigest() - - -def export(): - _ensure_out_dir() - state = _load_state() - since = state.get("since") # ISO8601 timestamp or None - - # GitHub "issues" list includes PRs; PRs have "pull_request" key. - issues_url = f"{BASE}/repos/{OWNER}/{REPO}/issues" - params = {"state": "all", "sort": "updated", "direction": "desc"} - if since: - params["since"] = since - - print("Fetching issues from:", issues_url, params) - - # Open output files once for streaming writes. - # Use append mode on incremental runs so previously exported records are - # preserved; use write mode on a full (first) run or when prior JSONL - # files are not present (e.g., fresh CI workspace) to avoid creating - # partial histories. Azure AI Search uses the stable document "id" for - # upserts, so duplicate records in append mode are safely overwritten by - # the indexer. - threads_path = os.path.join(OUT_DIR, "threads_issues_prs.jsonl") - comments_path = os.path.join(OUT_DIR, "issue_pr_comments.jsonl") - reviews_path = os.path.join(OUT_DIR, "pr_reviews.jsonl") - review_comments_path = os.path.join(OUT_DIR, "pr_review_comments.jsonl") - - if since and all(os.path.exists(p) for p in (threads_path, comments_path, reviews_path, review_comments_path)): - file_mode = "a" - else: - file_mode = "w" - newest = None - thread_count = 0 - - with open(threads_path, file_mode, encoding="utf-8") as f_threads, \ - open(comments_path, file_mode, encoding="utf-8") as f_comments, \ - open(reviews_path, file_mode, encoding="utf-8") as f_reviews, \ - open(review_comments_path, file_mode, encoding="utf-8") as f_review_comments: - - for it in _paginate(issues_url, params): - thread_count += 1 - number = it["number"] - is_pr = "pull_request" in it - print("Fetching id: ", number, " PR: ", is_pr) - - labels = [] - if isinstance(it.get("labels"), list): - labels = [l.get("name") for l in it.get("labels", []) if isinstance(l, dict) and l.get("name")] - - # 1) thread document (issue or PR) — written immediately - f_threads.write(json.dumps({ - "id": _stable_id("thread", f"{OWNER}/{REPO}#{number}"), - "content_type": "github_pr" if is_pr else "github_issue", - "repo": f"{OWNER}/{REPO}", - "github_number": number, - "title": it.get("title", ""), - "content": it.get("body") or "", - "state": it.get("state"), - "labels": labels, - "author": (it.get("user") or {}).get("login"), - "created_at": it.get("created_at"), - "updated_at": it.get("updated_at"), - "closed_at": it.get("closed_at"), - "source_url": it.get("html_url"), - }, ensure_ascii=False) + "\n") - - # Track newest timestamp for incremental state update - ts = it.get("updated_at") - if ts and (newest is None or ts > newest): - newest = ts - - # 2) issue comments (for issues AND PR timeline comments) - ic_url = f"{BASE}/repos/{OWNER}/{REPO}/issues/{number}/comments" - for c in _paginate(ic_url, {"since": since} if since else None): - f_comments.write(json.dumps({ - "id": _stable_id("issue_comment", number, c["id"]), - "content_type": "github_pr_issue_comment" if is_pr else "github_issue_comment", - "repo": f"{OWNER}/{REPO}", - "github_number": number, - "comment_id": c["id"], - "author": (c.get("user") or {}).get("login"), - "created_at": c.get("created_at"), - "updated_at": c.get("updated_at"), - "source_url": c.get("html_url"), - "content": c.get("body") or "", - }, ensure_ascii=False) + "\n") - - if is_pr: - # 3) PR reviews (top-level review bodies) - # The GitHub reviews endpoint does not support a 'since' filter; - # skip reviews whose submitted_at is before the incremental cursor. - reviews_url = f"{BASE}/repos/{OWNER}/{REPO}/pulls/{number}/reviews" - for rv in _paginate(reviews_url): - submitted_at = rv.get("submitted_at") - if since and submitted_at and submitted_at < since: - continue - f_reviews.write(json.dumps({ - "id": _stable_id("pr_review", number, rv["id"]), - "content_type": "github_pr_review", - "repo": f"{OWNER}/{REPO}", - "github_number": number, - "review_id": rv["id"], - "state": rv.get("state"), - "author": (rv.get("user") or {}).get("login"), - "submitted_at": submitted_at, - "source_url": rv.get("html_url"), - "content": rv.get("body") or "", - }, ensure_ascii=False) + "\n") - - # 4) PR review comments (diff comments) - # The GitHub pull review comments endpoint supports 'since', - # so pass the incremental cursor to avoid re-fetching old records. - prc_url = f"{BASE}/repos/{OWNER}/{REPO}/pulls/{number}/comments" - prc_params = {"since": since} if since else {} - for rc in _paginate(prc_url, prc_params): - f_review_comments.write(json.dumps({ - "id": _stable_id("pr_review_comment", number, rc["id"]), - "content_type": "github_pr_review_comment", - "repo": f"{OWNER}/{REPO}", - "github_number": number, - "comment_id": rc["id"], - "pull_request_review_id": rc.get("pull_request_review_id"), - "path": rc.get("path"), - "line": rc.get("line"), - "side": rc.get("side"), - "author": (rc.get("user") or {}).get("login"), - "created_at": rc.get("created_at"), - "updated_at": rc.get("updated_at"), - "source_url": rc.get("html_url"), - "content": rc.get("body") or "", - }, ensure_ascii=False) + "\n") - - # Flush after each thread so progress is preserved on interruption - f_threads.flush() - f_comments.flush() - f_reviews.flush() - f_review_comments.flush() - - print("Threads fetched:", thread_count) - - # Update incremental marker: latest updated_at seen - if newest: - state["since"] = newest - _save_state(state) - - print(f"[ok] wrote exports to: {OUT_DIR}") - print(f"[ok] next incremental since: {state.get('since')}") - - -if __name__ == "__main__": - export() \ No newline at end of file diff --git a/scripts/fetch_public_docs.py b/scripts/fetch_public_docs.py deleted file mode 100644 index 4e846fb98..000000000 --- a/scripts/fetch_public_docs.py +++ /dev/null @@ -1,30 +0,0 @@ -import requests -from pathlib import Path -import html2text - -DOCS = { - "blobfuse2-what-is.md": - "https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-what-is", - "blobfuse2-installation.md": - "https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-install?tabs=Ubuntu", - "blobfuse2-mount.md": - "https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-mount-container", - "blobfuse2-compare.md": - "https://learn.microsoft.com/en-us/azure/storage/blobs/blobfuse2-compare-linux-file-system" -} - -out_dir = Path("public") -out_dir.mkdir(exist_ok=True) - -converter = html2text.HTML2Text() -converter.ignore_links = False - -for name, url in DOCS.items(): - print(f"[public-docs] fetching {url}") - r = requests.get(url, timeout=30) - r.raise_for_status() - - md = converter.handle(r.text) - (out_dir / name).write_text(md, encoding="utf-8") - -print("[public-docs] done") \ No newline at end of file diff --git a/scripts/issueMetrics.py b/scripts/issueMetrics.py deleted file mode 100644 index 296088fc2..000000000 --- a/scripts/issueMetrics.py +++ /dev/null @@ -1,366 +0,0 @@ -import datetime -import argparse -from collections import Counter -import os -import statistics -import subprocess -import sys - -from github import Auth, Github -from github.GithubException import BadCredentialsException, GithubException - - -DEFAULT_DAYS_LOOKBACK = 90 - - -def get_env_var(*names: str) -> str: - for name in names: - value = os.getenv(name, "").strip() - if value: - return value - return "" - - -def get_gh_cli_token() -> str: - try: - result = subprocess.run( - ["gh", "auth", "token"], - check=True, - capture_output=True, - text=True, - ) - except FileNotFoundError: - return "" - except subprocess.CalledProcessError: - result = None - - if result is not None: - token = result.stdout.strip() - if token: - return token - - try: - result = subprocess.run( - ["gh", "auth", "status", "--show-token"], - check=True, - capture_output=True, - text=True, - ) - except FileNotFoundError: - return "" - except subprocess.CalledProcessError: - return "" - - token_output = "\n".join([result.stdout, result.stderr]) - for line in token_output.splitlines(): - marker = "Token:" - if marker in line: - token = line.split(marker, 1)[1].strip() - if token: - return token - return "" - - -def get_gh_cli_repo() -> str: - try: - result = subprocess.run( - ["gh", "repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"], - check=True, - capture_output=True, - text=True, - ) - except FileNotFoundError: - return "" - except subprocess.CalledProcessError: - return "" - return result.stdout.strip() - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Generate GitHub issue metrics report.") - parser.add_argument( - "--days", - type=int, - default=int(os.getenv("DAYS_LOOKBACK", str(DEFAULT_DAYS_LOOKBACK))), - help=f"Number of days to look back (default: {DEFAULT_DAYS_LOOKBACK}).", - ) - parser.add_argument( - "--only-copilot-tagged", - action="store_true", - help="Show only copilot-tagged entries in issue and PR details tables.", - ) - parser.add_argument( - "--summary", - action="store_true", - help="Show only summary tables (hide issue and PR details).", - ) - return parser.parse_args() - - -def print_table(title: str, headers: list[str], rows: list[list[str]]) -> None: - table_rows = [headers] + rows - widths = [max(len(str(row[index])) for row in table_rows) for index in range(len(headers))] - - def format_row(row: list[str]) -> str: - return "| " + " | ".join(str(value).ljust(widths[index]) for index, value in enumerate(row)) + " |" - - separator = "+-" + "-+-".join("-" * width for width in widths) + "-+" - - print(title) - print(separator) - print(format_row(headers)) - print(separator) - for row in rows: - print(format_row(row)) - print(separator) - - -def format_duration(hours: float) -> str: - if hours >= 24: - return f"{hours / 24:.2f} days" - return f"{hours:.2f} hrs" - - -def format_days(hours: float) -> str: - return f"{hours / 24:.2f}" - - -def truncate_text(text: str, max_length: int = 60) -> str: - if len(text) <= max_length: - return text - return text[: max_length - 3] + "..." - - -def is_copilot_tagged(labels) -> bool: - for label in labels: - if hasattr(label, "name") and "copilot" in label.name.lower(): - return True - return False - - -def get_first_comment_time_display_for_pr(pr) -> str: - first_comment_time = None - - try: - issue_comment = next(iter(pr.get_issue_comments()), None) - if issue_comment is not None and issue_comment.created_at is not None: - first_comment_time = issue_comment.created_at - except GithubException: - return "Unavailable" - - try: - review_comment = next(iter(pr.get_comments()), None) - if review_comment is not None and review_comment.created_at is not None: - if first_comment_time is None or review_comment.created_at < first_comment_time: - first_comment_time = review_comment.created_at - except GithubException: - return "Unavailable" - - if first_comment_time is None: - return "No comments" - - first_comment_hours = (first_comment_time - pr.created_at).total_seconds() / 3600 - return format_duration(first_comment_hours) - - -def main() -> int: - args = parse_args() - - access_token = get_env_var("GITHUB_TOKEN", "GH_TOKEN") or get_gh_cli_token() - repo_name = get_env_var("REPO_NAME", "GITHUB_REPOSITORY") or get_gh_cli_repo() - days_lookback = args.days - - if days_lookback <= 0: - print("Error: --days must be a positive integer.", file=sys.stderr) - return 1 - - if not access_token: - print( - "Error: Missing token. Set GITHUB_TOKEN/GH_TOKEN or run 'gh auth login'.", - file=sys.stderr, - ) - return 1 - - if not repo_name or "/" not in repo_name: - print("Error: Missing repo. Set REPO_NAME or GITHUB_REPOSITORY as 'owner/repo'.", file=sys.stderr) - return 1 - - try: - g = Github(auth=Auth.Token(access_token)) - repo = g.get_repo(repo_name) - except BadCredentialsException: - print("Error: GitHub authentication failed (bad token).", file=sys.stderr) - return 1 - except GithubException as ex: - print(f"Error: Unable to access repository '{repo_name}': {ex}", file=sys.stderr) - return 1 - - run_timestamp = datetime.datetime.now(datetime.timezone.utc) - since = run_timestamp - datetime.timedelta(days=days_lookback) - - print(f"Report Generated On: {run_timestamp:%Y-%m-%d %H:%M:%S} UTC") - print(f"Report Window: Last {days_lookback} days") - print() - - issues = repo.get_issues(state="all", since=since) - - new_count = 0 - open_count = 0 - resolved_count = 0 - issue_copilot_yes_count = 0 - resolution_times = [] - issue_rows = [] - - for issue in issues: - if issue.pull_request: - continue - - new_count += 1 - close_time_display = "Open" - first_comment_display = "No comments" - - if issue.comments and issue.comments > 0: - try: - first_comment = next(iter(issue.get_comments()), None) - if first_comment is not None and first_comment.created_at is not None: - first_comment_hours = (first_comment.created_at - issue.created_at).total_seconds() / 3600 - first_comment_display = format_duration(first_comment_hours) - except GithubException: - first_comment_display = "Unavailable" - - if issue.state == "open": - open_count += 1 - elif issue.state == "closed" and issue.closed_at is not None: - resolved_count += 1 - duration = issue.closed_at - issue.created_at - resolution_hours = duration.total_seconds() / 3600 - resolution_times.append(resolution_hours) - close_time_display = format_days(resolution_hours) - - issue_copilot_tag = "Yes" if is_copilot_tagged(issue.labels) else "No" - if issue_copilot_tag == "Yes": - issue_copilot_yes_count += 1 - - if args.only_copilot_tagged and issue_copilot_tag != "Yes": - continue - - issue_rows.append( - [ - f"#{issue.number}", - truncate_text(issue.title), - close_time_display, - first_comment_display, - issue_copilot_tag, - ] - ) - - if resolution_times: - mean_time = statistics.mean(resolution_times) - min_time = min(resolution_times) - max_time = max(resolution_times) - else: - mean_time = min_time = max_time = 0 - - print_table( - f"Issue Report (Last {days_lookback} Days)", - ["Repository", "Total New", "Still Open", "Resolved", "Copilot"], - [[repo_name, str(new_count), str(open_count), str(resolved_count), str(issue_copilot_yes_count)]], - ) - print() - print_table( - "Resolution Metrics", - ["Mean", "Min", "Max"], - [[format_duration(mean_time), format_duration(min_time), format_duration(max_time)]], - ) - if not args.summary: - print() - print_table( - "Issue Details", - ["Issue", "Title", "Days to Close", "Time to First Comment", "Copilot"], - issue_rows, - ) - - pull_requests = repo.get_pulls(state="all", sort="created", direction="desc") - - pr_created_count = 0 - pr_merged_count = 0 - pr_abandoned_count = 0 - pr_copilot_yes_count = 0 - pr_owner_counts = Counter() - pr_rows = [] - - for pr in pull_requests: - if pr.created_at < since: - break - - if pr.base is None or pr.base.ref != "main": - continue - - pr_created_count += 1 - time_to_close_display = "Open" - - if pr.state == "closed": - if pr.merged_at is not None: - pr_merged_count += 1 - else: - pr_abandoned_count += 1 - - if pr.closed_at is not None: - close_hours = (pr.closed_at - pr.created_at).total_seconds() / 3600 - time_to_close_display = format_duration(close_hours) - - pr_author = pr.user.login if pr.user else "Unknown" - pr_owner_counts[pr_author] += 1 - pr_author_normalized = pr_author.lower() - pr_copilot_tag = "Yes" if pr_author_normalized in {"dependabot[bot]", "copilot"} else "No" - if pr_copilot_tag != "Yes": - try: - pr_issue = repo.get_issue(pr.number) - pr_copilot_tag = "Yes" if is_copilot_tagged(pr_issue.labels) else "No" - except GithubException: - pr_copilot_tag = "No" - if pr_copilot_tag == "Yes": - pr_copilot_yes_count += 1 - - if args.only_copilot_tagged and pr_copilot_tag != "Yes": - continue - - pr_rows.append( - [ - f"#{pr.number}", - truncate_text(pr.title), - pr_author, - get_first_comment_time_display_for_pr(pr), - time_to_close_display, - pr_copilot_tag, - ] - ) - - print() - print_table( - f"PR Report (Last {days_lookback} Days)", - ["Repository", "PRs Created", "PRs Merged", "PRs Abandoned", "Copilot"], - [[repo_name, str(pr_created_count), str(pr_merged_count), str(pr_abandoned_count), str(pr_copilot_yes_count)]], - ) - - pr_owner_rows = [[owner, str(count)] for owner, count in pr_owner_counts.most_common()] - print() - print_table( - "PR Owner Summary", - ["Author", "PRs"], - pr_owner_rows, - ) - - if not args.summary: - print() - print_table( - "PR Details", - ["PR", "Title", "Author", "Time to First Comment", "Time to Merge/Close", "Copilot"], - pr_rows, - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file From 61f4f3a3b36d49fa8a93479da5004cb5cbcee2bb Mon Sep 17 00:00:00 2001 From: syeleti-msft Date: Wed, 24 Jun 2026 12:53:56 +0530 Subject: [PATCH 75/95] attr_cache: replace entry-count limit with memory-bounded LRU and background TTL sweeper (#2245) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 6 + common/cache_policy/lru_cache.go | 289 ++++++ common/cache_policy/lru_cache_test.go | 599 ++++++++++++ common/util.go | 10 + common/util_test.go | 10 + component/attr_cache/attr_cache.go | 557 +++++------ .../attr_cache/attr_cache_integration_test.go | 921 ++++++++++++++++++ component/attr_cache/attr_cache_test.go | 577 +++++++---- component/attr_cache/cacheMap.go | 111 --- component/attr_cache/cache_item.go | 236 +++++ component/attr_cache/cache_item_test.go | 497 ++++++++++ sampleBlockCacheConfig.yaml | 4 + sampleFileCacheConfig.yaml | 4 + sampleFileCacheWithSASConfig.yaml | 4 + setup/advancedConfig.yaml | 1 + setup/baseConfig.yaml | 5 + 16 files changed, 3178 insertions(+), 653 deletions(-) create mode 100644 common/cache_policy/lru_cache.go create mode 100644 common/cache_policy/lru_cache_test.go create mode 100644 component/attr_cache/attr_cache_integration_test.go delete mode 100644 component/attr_cache/cacheMap.go create mode 100644 component/attr_cache/cache_item.go create mode 100644 component/attr_cache/cache_item_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e699460f..797f1b2ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ ## 2.5.4 (Unreleased) **Features** - Make Blobfuse2 binary FIPS compliant by building with the Microsoft Go toolchain (`systemcrypto` GOEXPERIMENT) and `CGO_ENABLED=1`, routing all `crypto/*` calls through the system OpenSSL FIPS provider ([PR #2226](https://github.com/Azure/azure-storage-fuse/pull/2226)) +- Attribute cache is now memory-bounded. The cache uses a least-recently-used (LRU) eviction policy and a background sweeper that reclaims memory from TTL-expired entries when the cache is idle. A new `max-size-mb` config parameter (or `--attr-cache-max-size-mb` CLI flag) controls the memory limit. By default the limit is auto-tuned to 1% of total system RAM, clamped to [64 MB, 1 GB] (~70 K–1.1 M file entries depending on machine size). Omitting `max-size-mb` or setting it to `0` both use the auto-tuned value. Increase the limit for workloads with large directory trees or many distinct file paths. Example configuration: + ```yaml + attr_cache: + timeout-sec: 120 + max-size-mb: 256 # set explicitly to override auto-tuning + ``` **Bug Fixes** - Fix CPK-encrypted blob attribute lookup duplicating the prefix path when subdirectory is configured ([PR #2199](https://github.com/Azure/azure-storage-fuse/pull/2199)) diff --git a/common/cache_policy/lru_cache.go b/common/cache_policy/lru_cache.go new file mode 100644 index 000000000..c4893d5fb --- /dev/null +++ b/common/cache_policy/lru_cache.go @@ -0,0 +1,289 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2026 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package cache_policy + +import ( + "container/list" + "sync" + "unsafe" +) + +// mapBucketOverhead estimates the per-entry overhead from Go's internal hash map structure. +// Go's map uses a bucket-based design; 64 B covers the amortised cost of key/value copies +// and tophash bookkeeping per entry. +const mapBucketOverhead = 64 + +// lruItem is the value stored inside each list.Element in an LRU[K,V]. +type lruItem[K comparable, V any] struct { + key K + val V + userSize int64 // sizeOf(key, val) frozen at insertion time +} + +// LRU is a memory-bounded, generic least-recently-used cache. +// +// It is safe for concurrent use by multiple goroutines. +// +// Memory accounting works in two layers: +// - perEntryOverhead: fixed structural bytes per entry (list.Element + lruItem + map +// bucket), computed once in NewLRU via unsafe.Sizeof for the concrete K/V types. +// - sizeOf(k, v): caller-provided estimate of the heap bytes for one key-value pair, +// covering any heap allocations reachable from k and v (e.g. string data, pointed-to +// structs). It should NOT include the structural overhead above. +type LRU[K comparable, V any] struct { + mu sync.RWMutex + items map[K]*list.Element + list *list.List + maxSize int64 // 0 = no limit; immutable after construction + currSize int64 + perEntryOverhead int64 + sizeOf func(K, V) int64 +} + +// NewLRU creates an LRU with the given memory limit (bytes) and size estimator. +// maxSize == 0 disables memory-based eviction. +func NewLRU[K comparable, V any](maxSize int64, sizeOf func(K, V) int64) *LRU[K, V] { + var zeroItem lruItem[K, V] + overhead := int64(unsafe.Sizeof(zeroItem)) + + int64(unsafe.Sizeof(list.Element{})) + + mapBucketOverhead + l := &LRU[K, V]{ + items: make(map[K]*list.Element), + list: list.New(), + sizeOf: sizeOf, + perEntryOverhead: overhead, + maxSize: maxSize, + } + return l +} + +// Put inserts or updates a key-value pair and promotes it to MRU. +// After insertion, LRU entries are evicted until currSize <= maxSize. +// Returns false (and leaves the cache unchanged) if the entry's size alone +// exceeds maxSize — oversized entries are rejected upfront so existing entries +// are never displaced by a key that can never fit. +func (l *LRU[K, V]) Put(key K, val V) bool { + l.mu.Lock() + defer l.mu.Unlock() + return l.put(key, val) +} + +// put is the lock-free inner implementation, called with l.mu write-locked. +func (l *LRU[K, V]) put(key K, val V) bool { + userSize := l.sizeOf(key, val) + // Reject entries that can never fit, so existing LRU contents are not displaced. + if l.maxSize > 0 && l.perEntryOverhead+userSize > l.maxSize { + return false + } + if elem, ok := l.items[key]; ok { + item := elem.Value.(*lruItem[K, V]) + l.currSize -= item.userSize + item.val = val + item.userSize = userSize + l.currSize += userSize + l.list.MoveToFront(elem) + l.evictIfNeeded() + return true + } + item := &lruItem[K, V]{key: key, val: val, userSize: userSize} + elem := l.list.PushFront(item) + l.items[key] = elem + l.currSize += l.perEntryOverhead + userSize + l.evictIfNeeded() + return true +} + +// Get retrieves a value and promotes it to MRU. Returns (zero, false) on miss. +func (l *LRU[K, V]) Get(key K) (V, bool) { + l.mu.Lock() + defer l.mu.Unlock() + if elem, ok := l.items[key]; ok { + l.list.MoveToFront(elem) + return elem.Value.(*lruItem[K, V]).val, true + } + var zero V + return zero, false +} + +// Peek retrieves a value without changing LRU order. Returns (zero, false) on miss. +func (l *LRU[K, V]) Peek(key K) (V, bool) { + l.mu.RLock() + defer l.mu.RUnlock() + if elem, ok := l.items[key]; ok { + return elem.Value.(*lruItem[K, V]).val, true + } + var zero V + return zero, false +} + +// Has reports whether key is present without changing LRU order. +func (l *LRU[K, V]) Has(key K) bool { + l.mu.RLock() + defer l.mu.RUnlock() + _, ok := l.items[key] + return ok +} + +// Delete removes an entry from the cache. No-op if the key is absent. +func (l *LRU[K, V]) Delete(key K) { + l.mu.Lock() + defer l.mu.Unlock() + l.delete(key) +} + +// delete is the lock-free inner implementation, called with l.mu write-locked. +func (l *LRU[K, V]) delete(key K) { + if elem, ok := l.items[key]; ok { + item := elem.Value.(*lruItem[K, V]) + l.currSize -= l.perEntryOverhead + item.userSize + delete(l.items, key) + l.list.Remove(elem) + } +} + +// Len returns the number of entries currently in the cache. +func (l *LRU[K, V]) Len() int { + l.mu.RLock() + defer l.mu.RUnlock() + return len(l.items) +} + +// Size returns the current estimated memory usage in bytes. +func (l *LRU[K, V]) Size() int64 { + l.mu.RLock() + defer l.mu.RUnlock() + return l.currSize +} + +// MaxSize returns the configured memory limit (0 = no limit). +// This value is immutable after construction and requires no locking. +func (l *LRU[K, V]) MaxSize() int64 { + return l.maxSize +} + +// PerEntryOverhead returns the structural bytes added per entry (useful for testing). +// This value is immutable after construction and requires no locking. +func (l *LRU[K, V]) PerEntryOverhead() int64 { + return l.perEntryOverhead +} + +// Range calls fn for every entry in MRU→LRU order. +// +// The cache is read-locked for the duration of the call. fn must not call any +// LRU methods (Put, Get, Delete, etc.) — doing so will deadlock. Mutating a +// value pointed to by V is safe as long as the caller manages value-level +// concurrency separately. +func (l *LRU[K, V]) Range(fn func(K, V) bool) { + l.mu.RLock() + defer l.mu.RUnlock() + for elem := l.list.Front(); elem != nil; elem = elem.Next() { + item := elem.Value.(*lruItem[K, V]) + if !fn(item.key, item.val) { + return + } + } +} + +// Purge removes all entries. +func (l *LRU[K, V]) Purge() { + l.mu.Lock() + defer l.mu.Unlock() + l.items = make(map[K]*list.Element) + l.list.Init() + l.currSize = 0 +} + +// DeleteIf removes all entries for which pred returns true. +// It holds the write lock for the entire scan — single pass, no intermediate allocation. +func (l *LRU[K, V]) DeleteIf(pred func(K, V) bool) { + l.mu.Lock() + defer l.mu.Unlock() + for elem := l.list.Front(); elem != nil; { + item := elem.Value.(*lruItem[K, V]) + next := elem.Next() + if pred(item.key, item.val) { + l.currSize -= l.perEntryOverhead + item.userSize + delete(l.items, item.key) + l.list.Remove(elem) + } + elem = next + } +} + +// ReplaceIf replaces the value of every entry for which pred returns true with newVal(key). +// The factory is called once per matching entry so each gets its own value. +// It holds the write lock for the entire scan — no entry matching the predicate can be +// inserted or removed by another goroutine between the scan and the replacements. +func (l *LRU[K, V]) ReplaceIf(pred func(K, V) bool, newVal func(K) V) { + l.mu.Lock() + defer l.mu.Unlock() + for elem := l.list.Front(); elem != nil; { + next := elem.Next() + item := elem.Value.(*lruItem[K, V]) + if pred(item.key, item.val) { + v := newVal(item.key) + newSize := l.sizeOf(item.key, v) + l.currSize += newSize - item.userSize + item.val = v + item.userSize = newSize + l.list.MoveToFront(elem) + } + elem = next + } + l.evictIfNeeded() +} + +// evictIfNeeded removes the least-recently-used entries until currSize <= maxSize. +// Must be called with l.mu write-locked. +func (l *LRU[K, V]) evictIfNeeded() { + if l.maxSize <= 0 { + return + } + for l.currSize > l.maxSize && l.list.Len() > 0 { + l.evictLast() + } +} + +// evictLast removes the least-recently-used entry. +// Must be called with l.mu write-locked. +func (l *LRU[K, V]) evictLast() { + elem := l.list.Back() + if elem == nil { + return + } + item := elem.Value.(*lruItem[K, V]) + l.currSize -= l.perEntryOverhead + item.userSize + delete(l.items, item.key) + l.list.Remove(elem) +} diff --git a/common/cache_policy/lru_cache_test.go b/common/cache_policy/lru_cache_test.go new file mode 100644 index 000000000..98b2e2b14 --- /dev/null +++ b/common/cache_policy/lru_cache_test.go @@ -0,0 +1,599 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2026 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package cache_policy + +import ( + "container/list" + "sync" + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type lruCacheTestSuite struct { + suite.Suite + assert *assert.Assertions +} + +func (s *lruCacheTestSuite) SetupTest() { + s.assert = assert.New(s.T()) +} + +// identity sizeOf: returns len(key) + len(val) for string caches. +func strSizeOf(k, v string) int64 { + return int64(len(k) + len(v)) +} + +// zeroSizeOf ignores key and value; useful when we want to reason about overhead only. +func zeroSizeOf[K comparable, V any](K, V) int64 { return 0 } + +// ---- basic operations ---- + +func (s *lruCacheTestSuite) TestBasicPutGet() { + lru := NewLRU[string, string](0, strSizeOf) + lru.Put("a", "1") + lru.Put("b", "2") + + v, ok := lru.Get("a") + s.assert.True(ok) + s.assert.Equal("1", v) + + v, ok = lru.Get("b") + s.assert.True(ok) + s.assert.Equal("2", v) + + _, ok = lru.Get("missing") + s.assert.False(ok) +} + +func (s *lruCacheTestSuite) TestPeekDoesNotPromote() { + lru := NewLRU[string, string](0, strSizeOf) + lru.Put("a", "1") + lru.Put("b", "2") // b is now MRU + + // Peek at "a" — should not promote it + v, ok := lru.Peek("a") + s.assert.True(ok) + s.assert.Equal("1", v) + + // Without any limit the order won't matter for eviction, but verify via Range (MRU first) + var order []string + lru.Range(func(k, _ string) bool { + order = append(order, k) + return true + }) + s.assert.Equal([]string{"b", "a"}, order) // b is still MRU +} + +func (s *lruCacheTestSuite) TestHas() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + lru.Put("x", 42) + s.assert.True(lru.Has("x")) + s.assert.False(lru.Has("y")) +} + +func (s *lruCacheTestSuite) TestDelete() { + lru := NewLRU[string, string](0, strSizeOf) + lru.Put("a", "1") + lru.Put("b", "2") + lru.Delete("a") + + s.assert.False(lru.Has("a")) + s.assert.True(lru.Has("b")) + s.assert.Equal(1, lru.Len()) +} + +func (s *lruCacheTestSuite) TestDeleteAbsent() { + lru := NewLRU[string, string](0, strSizeOf) + // Should not panic + lru.Delete("nonexistent") + s.assert.Equal(0, lru.Len()) +} + +func (s *lruCacheTestSuite) TestLen() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + s.assert.Equal(0, lru.Len()) + lru.Put("a", 1) + s.assert.Equal(1, lru.Len()) + lru.Put("b", 2) + s.assert.Equal(2, lru.Len()) + lru.Delete("a") + s.assert.Equal(1, lru.Len()) +} + +func (s *lruCacheTestSuite) TestPurge() { + lru := NewLRU[string, string](0, strSizeOf) + lru.Put("a", "1") + lru.Put("b", "2") + lru.Purge() + s.assert.Equal(0, lru.Len()) + s.assert.Equal(int64(0), lru.Size()) + s.assert.False(lru.Has("a")) +} + +// ---- size accounting ---- + +func (s *lruCacheTestSuite) TestSizeAccounting() { + lru := NewLRU[string, string](0, strSizeOf) + overhead := lru.PerEntryOverhead() + + lru.Put("ab", "cd") // userSize = 2+2 = 4 + s.assert.Equal(overhead+4, lru.Size()) + + lru.Put("ab", "cdef") // update: userSize changes from 4 to 6 + s.assert.Equal(overhead+6, lru.Size()) + + lru.Delete("ab") + s.assert.Equal(int64(0), lru.Size()) +} + +func (s *lruCacheTestSuite) TestPerEntryOverheadUsesUnsafeSizeof() { + lru := NewLRU[string, string](0, strSizeOf) + overhead := lru.PerEntryOverhead() + + // Overhead must cover at least: lruItem struct + list.Element struct + var item lruItem[string, string] + minExpected := int64(unsafe.Sizeof(item)) + int64(unsafe.Sizeof(list.Element{})) + s.assert.GreaterOrEqual(overhead, minExpected, + "perEntryOverhead should be >= sizeof(lruItem) + sizeof(list.Element)") + // And must include the map bucket overhead constant + s.assert.GreaterOrEqual(overhead, minExpected+mapBucketOverhead) +} + +// ---- eviction ---- + +func (s *lruCacheTestSuite) TestNoLimitNoEviction() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) // maxSize=0 means no limit + for i := 0; i < 10000; i++ { + k := string(rune('a' + i%26)) + lru.Put(k, i) + } + // All distinct keys are "a".."z" (26 entries), none evicted + s.assert.Equal(26, lru.Len()) +} + +func (s *lruCacheTestSuite) TestEvictionAtCapacity() { + // Allow room for exactly 2 entries (overhead + 0 user bytes each) + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + overhead := lru.PerEntryOverhead() + lru = NewLRU[string, int](overhead*2, zeroSizeOf[string, int]) + + lru.Put("a", 1) + lru.Put("b", 2) + s.assert.Equal(2, lru.Len()) + + // Adding "c" should evict "a" (LRU) + lru.Put("c", 3) + s.assert.Equal(2, lru.Len()) + s.assert.False(lru.Has("a"), "LRU entry 'a' should have been evicted") + s.assert.True(lru.Has("b")) + s.assert.True(lru.Has("c")) +} + +func (s *lruCacheTestSuite) TestEvictionPromotion() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + overhead := lru.PerEntryOverhead() + lru = NewLRU[string, int](overhead*2, zeroSizeOf[string, int]) + + lru.Put("a", 1) + lru.Put("b", 2) // order: b(MRU), a(LRU) + + // Promote "a" via Get + lru.Get("a") // order: a(MRU), b(LRU) + + // Adding "c" should evict "b" (now LRU), not "a" + lru.Put("c", 3) + s.assert.Equal(2, lru.Len()) + s.assert.True(lru.Has("a"), "'a' was recently accessed and should survive") + s.assert.False(lru.Has("b"), "'b' should be evicted (LRU)") + s.assert.True(lru.Has("c")) +} + +func (s *lruCacheTestSuite) TestEvictionOrder() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + overhead := lru.PerEntryOverhead() + lru = NewLRU[string, int](overhead*3, zeroSizeOf[string, int]) + + lru.Put("a", 1) // LRU order: a + lru.Put("b", 2) // LRU order: b, a + lru.Put("c", 3) // LRU order: c, b, a — full + + lru.Put("d", 4) // evicts "a" + s.assert.False(lru.Has("a")) + lru.Put("e", 5) // evicts "b" + s.assert.False(lru.Has("b")) + + s.assert.True(lru.Has("c")) + s.assert.True(lru.Has("d")) + s.assert.True(lru.Has("e")) +} + +// ---- MaxSize accessor ---- + +func (s *lruCacheTestSuite) TestMaxSizeAccessor() { + lru := NewLRU[string, int](1024, zeroSizeOf[string, int]) + s.assert.Equal(int64(1024), lru.MaxSize()) +} + +// ---- oversized entry rejection ---- + +func (s *lruCacheTestSuite) TestOversizedEntryRejected() { + lru := NewLRU[string, string](0, strSizeOf) + overhead := lru.PerEntryOverhead() + lru = NewLRU[string, string](overhead+4, strSizeOf) // room for 1 entry with 4 user bytes + + lru.Put("ab", "cd") // userSize=4, fits exactly + s.assert.Equal(1, lru.Len()) + + // Update with a value too large to ever fit — rejected upfront, original entry survives. + retained := lru.Put("ab", "cdefgh") // userSize=8, exceeds limit + s.assert.False(retained, "entry whose size exceeds maxSize should be rejected") + s.assert.Equal(1, lru.Len(), "existing entry must not be displaced") + v, ok := lru.Get("ab") + s.assert.True(ok) + s.assert.Equal("cd", v, "original value must be unchanged") +} + +// ---- Range ---- + +func (s *lruCacheTestSuite) TestRange() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + lru.Put("a", 1) + lru.Put("b", 2) + lru.Put("c", 3) // MRU order: c, b, a + + var keys []string + lru.Range(func(k string, _ int) bool { + keys = append(keys, k) + return true + }) + s.assert.Equal([]string{"c", "b", "a"}, keys) +} + +func (s *lruCacheTestSuite) TestRangeEarlyStop() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + for _, k := range []string{"a", "b", "c", "d"} { + lru.Put(k, 1) + } + + count := 0 + lru.Range(func(_ string, _ int) bool { + count++ + return count < 2 // stop after 2 + }) + s.assert.Equal(2, count) +} + +// ---- integration: integer keys ---- + +func (s *lruCacheTestSuite) TestIntKeys() { + sizeOf := func(k int, v []byte) int64 { return int64(len(v)) } + lru := NewLRU[int, []byte](0, sizeOf) + lru.Put(1, []byte("hello")) + lru.Put(2, []byte("world")) + + v, ok := lru.Get(1) + s.assert.True(ok) + s.assert.Equal([]byte("hello"), v) + s.assert.Equal(2, lru.Len()) +} + +// ---- frozen userSize ---- + +func (s *lruCacheTestSuite) TestFrozenUserSizeOnMutation() { + // sizeOf reports the length of the slice the pointer points to. + // After insertion, we grow the slice — sizeOf would return a larger value, + // but the LRU must still use the size frozen at insertion time. + type buf struct{ data []byte } + sizeOf := func(_ string, v *buf) int64 { return int64(len(v.data)) } + + lru := NewLRU[string, *buf](0, sizeOf) + overhead := lru.PerEntryOverhead() + + b := &buf{data: []byte("hi")} // userSize = 2 at insertion + lru.Put("k", b) + s.assert.Equal(overhead+2, lru.Size()) + + // Mutate the pointed-to value — sizeOf would now return 100 + b.data = make([]byte, 100) + // Size must remain overhead+2 (frozen at insertion) + s.assert.Equal(overhead+2, lru.Size()) + + // Delete must subtract the frozen size, not the current size + lru.Delete("k") + s.assert.Equal(int64(0), lru.Size()) +} + +func (s *lruCacheTestSuite) TestFrozenUserSizeOnUpdate() { + // When Put is called again for an existing key the new userSize replaces the old one. + type buf struct{ data []byte } + sizeOf := func(_ string, v *buf) int64 { return int64(len(v.data)) } + + lru := NewLRU[string, *buf](0, sizeOf) + overhead := lru.PerEntryOverhead() + + b := &buf{data: []byte("hi")} + lru.Put("k", b) + + // Update with a value that reports a different size at Put time + b2 := &buf{data: make([]byte, 50)} + lru.Put("k", b2) + s.assert.Equal(overhead+50, lru.Size()) + + // Now mutate b2 — size must stay at 50 + b2.data = make([]byte, 200) + s.assert.Equal(overhead+50, lru.Size()) + + lru.Delete("k") + s.assert.Equal(int64(0), lru.Size()) +} + +// ---- delete size accounting ---- + +func (s *lruCacheTestSuite) TestDeleteSizeAccounting() { + lru := NewLRU[string, string](0, strSizeOf) + overhead := lru.PerEntryOverhead() + + lru.Put("ab", "cd") // userSize = 4 + s.assert.Equal(overhead+4, lru.Size()) + + lru.Delete("ab") + s.assert.Equal(int64(0), lru.Size()) + s.assert.Equal(0, lru.Len()) +} + +// ---- DeleteIf ---- + +func (s *lruCacheTestSuite) TestDeleteIf() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + overhead := lru.PerEntryOverhead() + + for _, k := range []string{"a", "b", "c", "d"} { + lru.Put(k, 1) + } + s.assert.Equal(int64(4)*overhead, lru.Size()) + + // Remove entries whose key >= "c" + lru.DeleteIf(func(k string, _ int) bool { return k >= "c" }) + + s.assert.Equal(2, lru.Len()) + s.assert.True(lru.Has("a")) + s.assert.True(lru.Has("b")) + s.assert.False(lru.Has("c")) + s.assert.False(lru.Has("d")) + s.assert.Equal(int64(2)*overhead, lru.Size()) +} + +func (s *lruCacheTestSuite) TestDeleteIfAll() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + lru.Put("a", 1) + lru.Put("b", 2) + + lru.DeleteIf(func(_ string, _ int) bool { return true }) + + s.assert.Equal(0, lru.Len()) + s.assert.Equal(int64(0), lru.Size()) +} + +func (s *lruCacheTestSuite) TestDeleteIfNone() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + overhead := lru.PerEntryOverhead() + lru.Put("a", 1) + lru.Put("b", 2) + + lru.DeleteIf(func(_ string, _ int) bool { return false }) + + s.assert.Equal(2, lru.Len()) + s.assert.Equal(int64(2)*overhead, lru.Size()) +} + +func (s *lruCacheTestSuite) TestDeleteIfEmpty() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + // Should not panic on empty cache + lru.DeleteIf(func(_ string, _ int) bool { return true }) + s.assert.Equal(0, lru.Len()) +} + +func (s *lruCacheTestSuite) TestReplaceIf() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + overhead := lru.PerEntryOverhead() + for _, k := range []string{"a", "b", "c", "d"} { + lru.Put(k, 1) + } + + lru.ReplaceIf(func(k string, _ int) bool { return k >= "c" }, func(_ string) int { return 99 }) + + s.assert.Equal(4, lru.Len()) + s.assert.Equal(int64(4)*overhead, lru.Size()) + v, ok := lru.Peek("a") + s.assert.True(ok) + s.assert.Equal(1, v) + v, ok = lru.Peek("c") + s.assert.True(ok) + s.assert.Equal(99, v) + v, ok = lru.Peek("d") + s.assert.True(ok) + s.assert.Equal(99, v) +} + +func (s *lruCacheTestSuite) TestReplaceIfAll() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + lru.Put("a", 1) + lru.Put("b", 2) + + lru.ReplaceIf(func(_ string, _ int) bool { return true }, func(_ string) int { return 0 }) + + s.assert.Equal(2, lru.Len()) + v, _ := lru.Peek("a") + s.assert.Equal(0, v) + v, _ = lru.Peek("b") + s.assert.Equal(0, v) +} + +func (s *lruCacheTestSuite) TestReplaceIfNone() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + overhead := lru.PerEntryOverhead() + lru.Put("a", 1) + lru.Put("b", 2) + + lru.ReplaceIf(func(_ string, _ int) bool { return false }, func(_ string) int { return 99 }) + + s.assert.Equal(2, lru.Len()) + s.assert.Equal(int64(2)*overhead, lru.Size()) + v, _ := lru.Peek("a") + s.assert.Equal(1, v) + v, _ = lru.Peek("b") + s.assert.Equal(2, v) +} + +func (s *lruCacheTestSuite) TestReplaceIfEmpty() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + // Should not panic on empty cache + lru.ReplaceIf(func(_ string, _ int) bool { return true }, func(_ string) int { return 0 }) + s.assert.Equal(0, lru.Len()) +} + +func (s *lruCacheTestSuite) TestReplaceIfPromotesToMRU() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + lru.Put("a", 1) + lru.Put("b", 2) // b is MRU + + lru.ReplaceIf(func(k string, _ int) bool { return k == "a" }, func(_ string) int { return 99 }) + + // "a" should have been promoted to MRU after replacement + var first string + lru.Range(func(k string, _ int) bool { first = k; return false }) + s.assert.Equal("a", first) +} + +// ---- purge then reuse ---- + +func (s *lruCacheTestSuite) TestPurgeAndReuse() { + lru := NewLRU[string, string](0, strSizeOf) + lru.Put("a", "1") + lru.Put("b", "2") + lru.Purge() + + // Cache must be fully functional after Purge + lru.Put("c", "3") + v, ok := lru.Get("c") + s.assert.True(ok) + s.assert.Equal("3", v) + s.assert.Equal(1, lru.Len()) + s.assert.False(lru.Has("a")) + s.assert.False(lru.Has("b")) +} + +// ---- Range on empty cache ---- + +func (s *lruCacheTestSuite) TestRangeOnEmptyCache() { + lru := NewLRU[string, int](0, zeroSizeOf[string, int]) + calls := 0 + lru.Range(func(_ string, _ int) bool { + calls++ + return true + }) + s.assert.Equal(0, calls) +} + +// ---- Peek on absent key ---- + +func (s *lruCacheTestSuite) TestPeekAbsent() { + lru := NewLRU[string, string](0, strSizeOf) + v, ok := lru.Peek("missing") + s.assert.False(ok) + s.assert.Empty(v) +} + +// ---- concurrent access ---- + +func (s *lruCacheTestSuite) TestConcurrentAccess() { + // Run under `go test -race` to detect data races. + lru := NewLRU[int, int](0, zeroSizeOf[int, int]) + const goroutines = 8 + const ops = 500 + + var wg sync.WaitGroup + wg.Add(goroutines) + for g := 0; g < goroutines; g++ { + go func(id int) { + defer wg.Done() + for i := 0; i < ops; i++ { + key := (id*ops + i) % 64 + lru.Put(key, id*ops+i) + lru.Get(key) + lru.Peek(key) + lru.Has(key) + if i%10 == 0 { + lru.Delete(key) + } + } + }(g) + } + wg.Wait() + // Just verify the LRU is in a consistent state + s.assert.GreaterOrEqual(lru.Len(), 0) + s.assert.GreaterOrEqual(lru.Size(), int64(0)) +} + +func (s *lruCacheTestSuite) TestConcurrentAccessWithLimit() { + lru := NewLRU[int, int](0, zeroSizeOf[int, int]) + overhead := lru.PerEntryOverhead() + lru = NewLRU[int, int](overhead*32, zeroSizeOf[int, int]) + + const goroutines = 8 + const ops = 500 + + var wg sync.WaitGroup + wg.Add(goroutines) + for g := 0; g < goroutines; g++ { + go func(id int) { + defer wg.Done() + for i := 0; i < ops; i++ { + key := (id*ops + i) % 128 + lru.Put(key, id*ops+i) + lru.Get(key) + } + }(g) + } + wg.Wait() + + s.assert.LessOrEqual(lru.Size(), overhead*32) +} + +func TestLRUCacheTestSuite(t *testing.T) { + suite.Run(t, new(lruCacheTestSuite)) +} diff --git a/common/util.go b/common/util.go index cd3be2bde..f16802728 100644 --- a/common/util.go +++ b/common/util.go @@ -649,3 +649,13 @@ func PrettyOpenFlags(f int) string { func GetGoroutineID() uint64 { return (uint64)(goid.Get()) } + +// TotalMemoryBytes returns the total physical RAM of the system in bytes. +// Returns 0 if the syscall fails. +func TotalMemoryBytes() uint64 { + var info syscall.Sysinfo_t + if err := syscall.Sysinfo(&info); err != nil { + return 0 + } + return uint64(info.Totalram) * uint64(info.Unit) +} diff --git a/common/util_test.go b/common/util_test.go index 924e80384..90d7f09f5 100644 --- a/common/util_test.go +++ b/common/util_test.go @@ -664,3 +664,13 @@ func (suite *utilTestSuite) TestSetFrsize() { SetFrsize(st, val) suite.assert.Equal(int64(val), st.Frsize) } + +func TestTotalMemoryBytesIsPositive(t *testing.T) { + mem := TotalMemoryBytes() + assert.Positive(t, mem, "TotalMemoryBytes should return a positive value on Linux") +} + +func TestTotalMemoryBytesAtLeast1MB(t *testing.T) { + mem := TotalMemoryBytes() + assert.GreaterOrEqual(t, mem, uint64(1<<20), "TotalMemoryBytes should be at least 1 MB") +} diff --git a/component/attr_cache/attr_cache.go b/component/attr_cache/attr_cache.go index 1fce85313..740c690ce 100644 --- a/component/attr_cache/attr_cache.go +++ b/component/attr_cache/attr_cache.go @@ -35,6 +35,7 @@ package attr_cache import ( "context" + "errors" "fmt" "os" "strings" @@ -42,36 +43,67 @@ import ( "syscall" "time" + "github.com/Azure/azure-storage-fuse/v2/common" "github.com/Azure/azure-storage-fuse/v2/common/config" "github.com/Azure/azure-storage-fuse/v2/common/log" "github.com/Azure/azure-storage-fuse/v2/internal" "github.com/Azure/azure-storage-fuse/v2/internal/handlemap" ) -// By default attr cache is valid for 120 seconds -const defaultAttrCacheTimeout uint32 = (120) +// defaultAttrCacheTimeout is the default TTL for cached attributes (seconds). +const defaultAttrCacheTimeout uint32 = 120 -// Common structure for AttrCache Component +const ( + defaultMaxSizeFloorMB int64 = 64 + defaultMaxSizeCeilingMB int64 = 1024 +) + +// defaultMaxSizeBytes returns the auto-tuned default memory limit for the attribute cache: +// 1% of total system RAM, clamped to [64 MB, 1 GB]. +// Using total RAM (not free RAM) gives a stable, reproducible value across reboots and +// concurrent mounts. +// +// Rough capacity at 64 MB (52-byte path, 20-byte ETag, 16-byte MD5): +// - negative entry (tombstone): ~320 B/entry → ~210 K entries +// - positive entry (no metadata): ~964 B/entry → ~70 K entries +// +// See TestNegativeEntryCapacityMatchesEstimate and TestPositiveEntryCapacityMatchesEstimate +// in cache_item_test.go for exact accounting and how to recalculate for different path lengths. +func defaultMaxSizeBytes() int64 { + totalMB := int64(common.TotalMemoryBytes() / (1024 * 1024)) + onePct := totalMB / 100 + if onePct < defaultMaxSizeFloorMB { + onePct = defaultMaxSizeFloorMB + } + if onePct > defaultMaxSizeCeilingMB { + onePct = defaultMaxSizeCeilingMB + } + return onePct * 1024 * 1024 +} + +// AttrCache is the pipeline component that caches file/directory attributes. +// The LRU is thread-safe; no additional locking is needed around individual operations. +// Expired entries are rejected lazily at GetAttr time; the memory-bounded LRU evicts +// stale entries by LRU order as new entries arrive. +// A background sweeper goroutine reclaims memory from TTL-expired entries when the cache is idle. type AttrCache struct { internal.BaseComponent - cacheTimeout uint32 - noSymlinks bool - maxFiles int - cacheMap map[string]*attrCacheItem - cacheLock sync.RWMutex - cleanupDone chan bool - cleanupCtx context.Context - cleanupStop context.CancelFunc + cacheTimeout time.Duration + noSymlinks bool + maxSizeBytes int64 + lru *attrCacheLRU + stopCh chan struct{} + sweepWg sync.WaitGroup + noCacheOnList bool } -// Structure defining your config parameters +// AttrCacheOptions holds the configuration for the attribute cache. type AttrCacheOptions struct { - Timeout uint32 `config:"timeout-sec" yaml:"timeout-sec,omitempty"` + Timeout uint32 `config:"timeout-sec" yaml:"timeout-sec,omitempty"` NoCacheOnList bool `config:"no-cache-on-list" yaml:"no-cache-on-list,omitempty"` - NoSymlinks bool `config:"no-symlinks" yaml:"no-symlinks,omitempty"` - - //maximum file attributes overall to be cached - MaxFiles int `config:"max-files" yaml:"max-files,omitempty"` + NoSymlinks bool `config:"no-symlinks" yaml:"no-symlinks,omitempty"` + MaxSizeMB uint32 `config:"max-size-mb" yaml:"max-size-mb,omitempty"` + MaxFiles uint32 `config:"max-files" yaml:"max-files,omitempty"` // support v1 CacheOnList bool `config:"cache-on-list"` @@ -79,10 +111,6 @@ type AttrCacheOptions struct { const compName = "attr_cache" -// caching only first 5 mil files by default -// caching more means increased memory usage of the process -const defaultMaxFiles = 5000000 // 5 million max files overall to be cached - // Verification to check satisfaction criteria with Component Interface var _ internal.Component = &AttrCache{} @@ -102,54 +130,98 @@ func (ac *AttrCache) Priority() internal.ComponentPriority { return internal.EComponentPriority.LevelTwo() } -// Start : Pipeline calls this method to start the component functionality -// -// this shall not block the call otherwise pipeline will not start -func (ac *AttrCache) Start(ctx context.Context) error { +// Start initialises the cache and launches the background TTL sweeper. +func (ac *AttrCache) Start(_ context.Context) error { log.Trace("AttrCache::Start : Starting component %s", ac.Name()) - - // AttrCache : start code goes here - ac.cacheMap = make(map[string]*attrCacheItem) - - // Start background cleanup goroutine - ac.cleanupCtx, ac.cleanupStop = context.WithCancel(ctx) - ac.cleanupDone = make(chan bool) - go ac.backgroundCleanup() - + ac.lru = newAttrCacheLRU(ac.maxSizeBytes) + if ac.cacheTimeout > 0 { + ac.stopCh = make(chan struct{}) + ac.sweepWg.Add(1) + go func() { + defer ac.sweepWg.Done() + ac.ttlSweeper() + }() + } return nil } -// Stop : Stop the component functionality and kill all threads started +// Stop cancels the background sweeper and waits for it to exit before returning. +// Memory held by expired entries is reclaimed by the GC once the component is released. func (ac *AttrCache) Stop() error { log.Trace("AttrCache::Stop : Stopping component %s", ac.Name()) + if ac.stopCh != nil { + close(ac.stopCh) + ac.sweepWg.Wait() + ac.stopCh = nil + } + return nil +} - // Stop the background cleanup goroutine - if ac.cleanupStop != nil { - ac.cleanupStop() - <-ac.cleanupDone // Wait for cleanup goroutine to finish +// ttlSweeper ticks every cacheTimeout and evicts expired entries when the cache is idle. +func (ac *AttrCache) ttlSweeper() { + ticker := time.NewTicker(ac.cacheTimeout) + defer ticker.Stop() + for { + select { + case <-ticker.C: + ac.sweepExpired() + case <-ac.stopCh: + return + } } +} - return nil +// sweepExpired removes all TTL-expired entries from the LRU. +// The idle gate skips the sweep if the cache was used within the last cacheTimeout/2 +// to avoid holding the write lock during active traffic. +func (ac *AttrCache) sweepExpired() { + if ac.cacheTimeout <= 0 { + return + } + + // Only apply the idle gate when the cache is memory-bounded. If MaxSize()==0 (unlimited), + // skipping sweeps under continuous traffic can let TTL-expired entries accumulate forever. + idleThreshold := ac.cacheTimeout / 2 + if idleThreshold > 0 && ac.lru.MaxSize() > 0 { + if last := ac.lru.lastOp.Load(); last != 0 { + if time.Since(time.Unix(last, 0)) < idleThreshold { + return + } + } + } + timeout := ac.cacheTimeout + cutoff := time.Now().Add(-timeout) + before := ac.lru.Size() + ac.lru.DeleteIf(func(_ string, item *attrCacheItem) bool { + return !item.cachedAt.After(cutoff) + }) + maxSize := ac.lru.MaxSize() + maxMB := (maxSize + (1 << 20) - 1) >> 20 + if maxSize == 0 { + log.Debug("AttrCache::sweepExpired : size %d MB (unbounded) (%d entries), reclaimed %d MB", + ac.lru.Size()>>20, ac.lru.Len(), (before-ac.lru.Size())>>20) + return + } + log.Debug("AttrCache::sweepExpired : size %d MB / %d MB (%d entries), reclaimed %d MB", + ac.lru.Size()>>20, maxMB, ac.lru.Len(), (before-ac.lru.Size())>>20) } -// GenConfig : Generate the default config for the component +// GenConfig returns a default configuration snippet for this component. func (ac *AttrCache) GenConfig() string { log.Info("AttrCache::Configure : config generation started") var sb strings.Builder fmt.Fprintf(&sb, "\n%s:", ac.Name()) fmt.Fprintf(&sb, "\n timeout-sec: %v", defaultAttrCacheTimeout) + fmt.Fprintf(&sb, "\n max-size-mb: %v", defaultMaxSizeBytes()/(1024*1024)) return sb.String() } -// Configure : Pipeline will call this method after constructor so that you can read config and initialize yourself -// -// Return failure if any config is not valid to exit the process +// Configure reads component configuration and applies it. func (ac *AttrCache) Configure(_ bool) error { log.Trace("AttrCache::Configure : %s", ac.Name()) - // >> If you do not need any config parameters remove below code and return nil conf := AttrCacheOptions{} err := config.UnmarshalKey(ac.Name(), &conf) if err != nil { @@ -158,178 +230,67 @@ func (ac *AttrCache) Configure(_ bool) error { } if config.IsSet(compName + ".timeout-sec") { - ac.cacheTimeout = conf.Timeout + ac.cacheTimeout = time.Duration(conf.Timeout) * time.Second } else { - ac.cacheTimeout = defaultAttrCacheTimeout - } - - if config.IsSet(compName + ".max-files") { - ac.maxFiles = conf.MaxFiles - } else { - ac.maxFiles = defaultMaxFiles + ac.cacheTimeout = time.Duration(defaultAttrCacheTimeout) * time.Second } if config.IsSet(compName + ".no-symlinks") { ac.noSymlinks = conf.NoSymlinks } - log.Crit("AttrCache::Configure : cache-timeout %d, symlink %t, max-files %d", - ac.cacheTimeout, ac.noSymlinks, ac.maxFiles) - - return nil -} - -// OnConfigChange : If component has registered, on config file change this method is called -func (ac *AttrCache) OnConfigChange() { - log.Trace("AttrCache::OnConfigChange : %s", ac.Name()) - _ = ac.Configure(true) -} - -// Helper Methods -// deleteDirectory: recursively marks a directory and its children from cache -// these entries are then marked as deleted to serve ENOENT responses. -func (ac *AttrCache) deleteDirectory(path string, time time.Time) { - // Recursively mark the children of the path as deleted, then delete the path - // For example, filesystem: a/, a/b, a/c, aa/, ab. - // When we delete directory a, we only want to delete a/, a/b, and a/c. - // If we do not conditionally extend a, we would accidentally delete aa/ and ab - - // Add a trailing / so that we only delete child paths under the directory and not paths that have the same prefix - prefix := internal.ExtendDirName(path) - - for key, value := range ac.cacheMap { - if strings.HasPrefix(key, prefix) { - value.markDeleted(time) - } + // no-cache-on-list (v2) takes priority; fall back to inverted cache-on-list (v1). + if config.IsSet(compName + ".no-cache-on-list") { + ac.noCacheOnList = conf.NoCacheOnList + } else if config.IsSet(compName + ".cache-on-list") { + ac.noCacheOnList = !conf.CacheOnList } - // We need to delete the path itself since we only handle children above. - ac.deletePath(path, time) -} - -// deletePath: removes a path from cache -func (ac *AttrCache) deletePath(path string, time time.Time) { - // Keys in the cache map do not contain trailing /, truncate the path before referencing a key in the map. - value, found := ac.cacheMap[internal.TruncateDirName(path)] - if found { - value.markDeleted(time) + if config.IsSet(compName+".max-size-mb") && conf.MaxSizeMB > 0 { + ac.maxSizeBytes = int64(conf.MaxSizeMB) * 1024 * 1024 + } else if config.IsSet(compName+".max-files") && conf.MaxFiles > 0 { + // max-files is deprecated; derive an approximate memory limit from the entry count + // using the positive-entry size estimate (~964 B/entry) and warn the user. + ac.maxSizeBytes = int64(conf.MaxFiles) * positiveEntrySize + log.Warn("AttrCache::Configure : 'max-files' is deprecated; derived max-size-mb=%d from max-files=%d. "+ + "Set 'max-size-mb' explicitly to silence this warning.", + ac.maxSizeBytes/(1024*1024), conf.MaxFiles) + } else { + ac.maxSizeBytes = defaultMaxSizeBytes() } -} -// invalidateDirectory: recursively marks a directory invalid -func (ac *AttrCache) invalidateDirectory(path string) { - // Recursively invalidate the children of the path, then invalidate the path - // For example, filesystem: a/, a/b, a/c, aa/, ab. - // When we invalidate directory a, we only want to invalidate a/, a/b, and a/c. - // If we do not conditionally extend a, we would accidentally invalidate aa/ and ab - - // Add a trailing / so that we only invalidate child paths under the directory and not paths that have the same prefix - prefix := internal.ExtendDirName(path) - - for key, value := range ac.cacheMap { - if strings.HasPrefix(key, prefix) { - value.invalidate() - } + // When timeout is zero entries are written but immediately considered expired, + // so cap the limit to 64 MB to avoid a large idle allocation. + if ac.cacheTimeout == 0 && ac.maxSizeBytes > defaultMaxSizeFloorMB*1024*1024 { + ac.maxSizeBytes = defaultMaxSizeFloorMB * 1024 * 1024 } - // We need to invalidate the path itself since we only handle children above. - ac.invalidatePath(path) -} - -// Copies the attr to the given path. -func (ac *AttrCache) updateCacheEntry(path string, attr *internal.ObjAttr) { - cacheEntry, found := ac.cacheMap[path] - if found { - // Copy the attr - cacheEntry.attr = attr - // Update the path inside the attr - cacheEntry.attr.Path = path - // Update the Existence of the entry - cacheEntry.attrFlag.Set(AttrFlagExists) - // Refresh the cache entry - cacheEntry.cachedAt = time.Now() - } -} + effectiveMB := uint32(ac.maxSizeBytes / (1024 * 1024)) + log.Crit("AttrCache::Configure : cache-timeout %v, no-symlinks %t, no-cache-on-list %t, max-size-mb %d", + ac.cacheTimeout, ac.noSymlinks, ac.noCacheOnList, effectiveMB) -// invalidatePath: invalidates a path -func (ac *AttrCache) invalidatePath(path string) { - // Keys in the cache map do not contain trailing /, truncate the path before referencing a key in the map. - value, found := ac.cacheMap[internal.TruncateDirName(path)] - if found { - value.invalidate() - } + return nil } -// backgroundCleanup: runs in a separate goroutine to periodically clean up expired entries -func (ac *AttrCache) backgroundCleanup() { - defer close(ac.cleanupDone) - - // Ensure minimum interval to prevent panic with NewTicker. - // Note: `cacheTimeout` is immutable post-start and should not be modified during runtime. - interval := time.Duration(ac.cacheTimeout) * time.Second - if interval <= 0 { - interval = time.Second // Use 1 second as minimum interval - } - - // Create ticker based on cache timeout interval - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ac.cleanupCtx.Done(): - log.Trace("AttrCache::backgroundCleanup : Stopping background cleanup") - return - case <-ticker.C: - ac.cleanupExpiredEntries() - } - } +// OnConfigChange logs that attr_cache settings cannot be applied safely at runtime. +func (ac *AttrCache) OnConfigChange() { + log.Warn("AttrCache::OnConfigChange : config change detected but not applied; restart required to apply new attr_cache settings") } -// cleanupExpiredEntries: removes expired entries from the cache map -// This runs in a background goroutine to prevent memory leaks -func (ac *AttrCache) cleanupExpiredEntries() { - // First pass: collect keys to delete under read lock to minimize write lock duration - var keysToDelete []string - ac.cacheLock.RLock() - for path, item := range ac.cacheMap { - // Check if entry has exceeded the cache timeout - if time.Since(item.cachedAt).Seconds() >= float64(ac.cacheTimeout) { - keysToDelete = append(keysToDelete, path) - } - } - ac.cacheLock.RUnlock() - - // Second pass: delete expired entries under write lock, re-checking expiration - if len(keysToDelete) > 0 { - ac.cacheLock.Lock() - for _, path := range keysToDelete { - // Re-check if entry still exists and is still expired - if item, exists := ac.cacheMap[path]; exists { - if time.Since(item.cachedAt).Seconds() >= float64(ac.cacheTimeout) { - delete(ac.cacheMap, path) - } - } - } - ac.cacheLock.Unlock() - } -} +// ------------------------- Component operations ------------------------------------------- -// ------------------------- Methods implemented by this component ------------------------------------------- -// CreateDir: Mark the directory invalid +// CreateDir marks the directory invalid in the cache. func (ac *AttrCache) CreateDir(options internal.CreateDirOptions) error { log.Trace("AttrCache::CreateDir : %s", options.Name) err := ac.NextComponent().CreateDir(options) if err == nil || err == syscall.EEXIST { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.invalidatePath(options.Name) + ac.lru.invalidatePath(options.Name) } return err } -// DeleteDir: Mark the directory deleted and recursively mark all it's children deleted +// DeleteDir marks the directory and all its children as deleted. func (ac *AttrCache) DeleteDir(options internal.DeleteDirOptions) error { log.Trace("AttrCache::DeleteDir : %s", options.Name) @@ -337,62 +298,37 @@ func (ac *AttrCache) DeleteDir(options internal.DeleteDirOptions) error { err := ac.NextComponent().DeleteDir(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.deleteDirectory(options.Name, deletionTime) + ac.lru.deleteDirectory(options.Name, deletionTime) } return err } -// ReadDir : Optionally cache attributes of paths returned by next component +// ReadDir caches attributes of all paths returned by the next component. func (ac *AttrCache) ReadDir(options internal.ReadDirOptions) (pathList []*internal.ObjAttr, err error) { log.Trace("AttrCache::ReadDir : %s", options.Name) pathList, err = ac.NextComponent().ReadDir(options) - if err == nil { - ac.cacheAttributes(pathList) + if err == nil && !ac.noCacheOnList { + ac.lru.cacheAttributes(pathList) } return pathList, err } -// StreamDir : Optionally cache attributes of paths returned by next component +// StreamDir caches attributes of all paths returned by the next component. func (ac *AttrCache) StreamDir(options internal.StreamDirOptions) ([]*internal.ObjAttr, string, error) { log.Trace("AttrCache::StreamDir : %s", options.Name) pathList, token, err := ac.NextComponent().StreamDir(options) - if err == nil { - ac.cacheAttributes(pathList) + if err == nil && !ac.noCacheOnList { + ac.lru.cacheAttributes(pathList) } return pathList, token, err } -// cacheAttributes : On dir listing cache the attributes for all files -func (ac *AttrCache) cacheAttributes(pathList []*internal.ObjAttr) { - // Check whether or not we are supposed to cache on list - if len(pathList) > 0 { - // Putting this inside loop is heavy as for each item we will do a kernel call to get current time - // If there are millions of blobs then cost of this is very high. - currTime := time.Now() - - for _, attr := range pathList { - if len(ac.cacheMap) > ac.maxFiles { - log.Debug("AttrCache::cacheAttributes : skipping adding path to attribute cache because it is full (size: %d, max: %d)", len(ac.cacheMap), ac.maxFiles) - break - } - - ac.cacheLock.Lock() - ac.cacheMap[internal.TruncateDirName(attr.Path)] = newAttrCacheItem(attr, true, currTime) - ac.cacheLock.Unlock() - } - - } -} - -// RenameDir : Mark the source directory deleted and recursively mark all it's children deleted. -// Invalidate the destination since we may have overwritten it. +// RenameDir marks the source directory deleted and invalidates the destination. func (ac *AttrCache) RenameDir(options internal.RenameDirOptions) error { log.Trace("AttrCache::RenameDir : %s -> %s", options.Src, options.Dst) @@ -400,71 +336,60 @@ func (ac *AttrCache) RenameDir(options internal.RenameDirOptions) error { err := ac.NextComponent().RenameDir(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.deleteDirectory(options.Src, deletionTime) + ac.lru.deleteDirectory(options.Src, deletionTime) // TLDR: Dst is guaranteed to be non-existent or empty. - // Note: We do not need to invalidate children of Dst due to the logic in our FUSE connector, see comments there, - // but it is always safer to double check than not. - ac.invalidateDirectory(options.Dst) + // Note: We do not need to invalidate children of Dst due to the logic in our FUSE + // connector, but it is always safer to double-check. + ac.lru.invalidateDirectory(options.Dst) } return err } -// CreateFile: Mark the file invalid +// CreateFile marks the file invalid in the cache. func (ac *AttrCache) CreateFile(options internal.CreateFileOptions) (*handlemap.Handle, error) { log.Trace("AttrCache::CreateFile : %s", options.Name) h, err := ac.NextComponent().CreateFile(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.invalidatePath(options.Name) + ac.lru.invalidatePath(options.Name) } return h, err } -// DeleteFile : Mark the file deleted +// DeleteFile marks the file as deleted in the cache. func (ac *AttrCache) DeleteFile(options internal.DeleteFileOptions) error { log.Trace("AttrCache::DeleteFile : %s", options.Name) err := ac.NextComponent().DeleteFile(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.deletePath(options.Name, time.Now()) + ac.lru.deletePath(options.Name, time.Now()) } return err } -// RenameFile : Mark the source file deleted. Invalidate the destination file. +// RenameFile copies source attributes to destination and marks the source deleted. func (ac *AttrCache) RenameFile(options internal.RenameFileOptions) error { log.Trace("AttrCache::RenameFile : %s -> %s", options.Src, options.Dst) srcAttr := options.SrcAttr err := ac.NextComponent().RenameFile(options) if err == nil { - // Copy source attribute to destination. - // LMT of Source will be modified by next component if the copy is success. - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.updateCacheEntry(options.Dst, srcAttr) - ac.deletePath(options.Src, time.Now()) + ac.lru.refreshEntry(options.Dst, srcAttr) + ac.lru.deletePath(options.Src, time.Now()) } return err } -// WriteFile : Mark the file invalid +// WriteFile retrieves metadata from the cache, forwards the write, then invalidates. func (ac *AttrCache) WriteFile(options *internal.WriteFileOptions) (int, error) { - // GetAttr on cache hit will serve from cache, on cache miss will serve from next component. attr, err := ac.GetAttr(internal.GetAttrOptions{Name: options.Handle.Path, RetrieveMetadata: true}) if err != nil { - // Ignore not exists errors - this can happen if createEmptyFile is set to false - if !os.IsNotExist(err) && err != syscall.ENOENT { + // Ignore not-exists errors — this can happen if createEmptyFile is set to false. + if !errors.Is(err, os.ErrNotExist) { return 0, err } } @@ -474,46 +399,30 @@ func (ac *AttrCache) WriteFile(options *internal.WriteFileOptions) (int, error) size, err := ac.NextComponent().WriteFile(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - // TODO: Could we just update the size and mod time of the file here? Or can other attributes change here? - ac.invalidatePath(options.Handle.Path) + ac.lru.invalidatePath(options.Handle.Path) } return size, err } -// TruncateFile : Update the file with its truncated size +// TruncateFile invalidates the cached entry so the next GetAttr fetches updated ETag/timestamps. func (ac *AttrCache) TruncateFile(options internal.TruncateFileOptions) error { log.Trace("AttrCache::TruncateFile : %s", options.Name) err := ac.NextComponent().TruncateFile(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - - // no need to truncate the name of the file - value, found := ac.cacheMap[options.Name] - if found && value.valid() && value.exists() { - value.setSize(options.NewSize) - } - // todo: invalidating path here rather than updating with etag - // due to some changes that are required in az storage comp which - // were not necessarily required. Once they were done invalidation - // of the attribute can be removed. - ac.invalidatePath(options.Name) + ac.lru.invalidatePath(options.Name) } return err } -// CopyFromFile : Mark the file invalid +// CopyFromFile retrieves metadata from the cache, forwards the copy, then invalidates. func (ac *AttrCache) CopyFromFile(options internal.CopyFromFileOptions) error { log.Trace("AttrCache::CopyFromFile : %s", options.Name) - // GetAttr on cache hit will serve from cache, on cache miss will serve from next component. attr, err := ac.GetAttr(internal.GetAttrOptions{Name: options.Name, RetrieveMetadata: true}) if err != nil { // Ignore not exists errors - this can happen if createEmptyFile is set to false - if !os.IsNotExist(err) && err != syscall.ENOENT { + if !errors.Is(err, os.ErrNotExist) { return err } } @@ -523,131 +432,115 @@ func (ac *AttrCache) CopyFromFile(options internal.CopyFromFileOptions) error { err = ac.NextComponent().CopyFromFile(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - // TODO: Could we just update the size and mod time of the file here? Or can other attributes change here? - // TODO: we're RLocking the cache but we need to also lock this attr item because another thread could be reading this attr item - ac.invalidatePath(options.Name) + ac.lru.invalidatePath(options.Name) } return err } +// SyncFile invalidates the cached entry after a sync. func (ac *AttrCache) SyncFile(options internal.SyncFileOptions) error { log.Trace("AttrCache::SyncFile : %s", options.Handle.Path) err := ac.NextComponent().SyncFile(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.invalidatePath(options.Handle.Path) + ac.lru.invalidatePath(options.Handle.Path) } return err } +// SyncDir recursively invalidates cached entries for a directory. func (ac *AttrCache) SyncDir(options internal.SyncDirOptions) error { log.Trace("AttrCache::SyncDir : %s", options.Name) err := ac.NextComponent().SyncDir(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.invalidateDirectory(options.Name) + ac.lru.invalidateDirectory(options.Name) } return err } -// GetAttr : Try to serve the request from the attribute cache, otherwise cache attributes of the path returned by next component +// GetAttr serves from cache on hit (promoting the entry to MRU), or fetches from the +// next component and caches the result on miss. func (ac *AttrCache) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr, error) { log.Trace("AttrCache::GetAttr : %s", options.Name) truncatedPath := internal.TruncateDirName(options.Name) - ac.cacheLock.RLock() - value, found := ac.cacheMap[truncatedPath] - ac.cacheLock.RUnlock() - - // Try to serve the request from the attribute cache - if found && value.valid() && time.Since(value.cachedAt).Seconds() < float64(ac.cacheTimeout) { - if value.isDeleted() { - log.Debug("AttrCache::GetAttr : %s served from cache", options.Name) - // no entry if path does not exist - return &internal.ObjAttr{}, syscall.ENOENT - } else { + if item, ok := ac.lru.Get(truncatedPath); ok { + if time.Since(item.cachedAt) < ac.cacheTimeout { log.Debug("AttrCache::GetAttr : %s served from cache", options.Name) - return value.getAttr(), nil + if item.isNegativeEntry() { + return &internal.ObjAttr{}, syscall.ENOENT + } + + if item.attr != nil { + return item.attr, nil + } + + log.Crit("AttrCache::GetAttr : %s is marked as positive entry in cache but attr is nil", truncatedPath) } } - // Get the attributes from next component and cache them + // Cache miss: fetch from next component. pathAttr, err := ac.NextComponent().GetAttr(options) - ac.cacheLock.Lock() - defer ac.cacheLock.Unlock() - - switch err { - case nil: - // Retrieved attributes so cache them - if len(ac.cacheMap) < ac.maxFiles { - ac.cacheMap[truncatedPath] = newAttrCacheItem(pathAttr, true, time.Now()) - } else { - log.Debug("AttrCache::GetAttr : %s skipping adding to attribute cache because it is full", options.Name) - } - case syscall.ENOENT: - // Path does not exist so cache a no-entry item - ac.cacheMap[truncatedPath] = newAttrCacheItem(&internal.ObjAttr{}, false, time.Now()) + if err == nil { + ac.lru.cachePositiveEntry(truncatedPath, pathAttr) + } else if errors.Is(err, os.ErrNotExist) { + // Cache negative entries so repeated lookups for absent paths are cheap. + // errors.Is matches syscall.ENOENT, *fs.PathError wrapping ENOENT, and bare os.ErrNotExist sentinels. + ac.lru.cacheNegativeEntry(truncatedPath) } return pathAttr, err } -// CreateLink : Mark the link and target invalid +// CreateLink marks the link and its target invalid in the cache. func (ac *AttrCache) CreateLink(options internal.CreateLinkOptions) error { log.Trace("AttrCache::CreateLink : Create symlink %s -> %s", options.Name, options.Target) err := ac.NextComponent().CreateLink(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - ac.invalidatePath(options.Name) - ac.invalidatePath(options.Target) // TODO : Why do we invalidate the target? Shouldn't the target remain unchanged? + ac.lru.invalidatePath(options.Name) + ac.lru.invalidatePath(options.Target) } return err } -// FlushFile : flush file +// FlushFile invalidates the cached entry after a flush. func (ac *AttrCache) FlushFile(options internal.FlushFileOptions) error { log.Trace("AttrCache::FlushFile : %s", options.Handle.Path) err := ac.NextComponent().FlushFile(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - - ac.invalidatePath(options.Handle.Path) + ac.lru.invalidatePath(options.Handle.Path) } return err } -// Chmod : Update the file with its new permissions +// Chmod updates the cached mode for a file or directory. +// It puts a new immutable item so concurrent readers see a consistent snapshot. func (ac *AttrCache) Chmod(options internal.ChmodOptions) error { log.Trace("AttrCache::Chmod : Change mode of file/directory %s", options.Name) err := ac.NextComponent().Chmod(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - - value, found := ac.cacheMap[internal.TruncateDirName(options.Name)] - if found && value.valid() && value.exists() { - value.setMode(options.Mode) + truncated := internal.TruncateDirName(options.Name) + if item, ok := ac.lru.Peek(truncated); ok && item.exists && item.attr != nil { + newAttr := *item.attr // copy the struct so the old item stays immutable + newAttr.Mode = options.Mode + newAttr.Ctime = time.Now() + if !ac.lru.Put(truncated, &attrCacheItem{attr: &newAttr, exists: true, cachedAt: time.Now()}) { + log.Err("AttrCache::Chmod : entry too large for cache, skipping path %s", truncated) + } } } return err } -// Chown : Update the file with its new owner and group (when datalake chown is implemented) +// Chown updates the file owner (when datalake chown is implemented). func (ac *AttrCache) Chown(options internal.ChownOptions) error { log.Trace("AttrCache::Chown : Change owner of file/directory %s", options.Name) @@ -657,22 +550,19 @@ func (ac *AttrCache) Chown(options internal.ChownOptions) error { return err } +// CommitData invalidates the cached entry after a data commit. func (ac *AttrCache) CommitData(options internal.CommitDataOptions) error { log.Trace("AttrCache::CommitData : %s", options.Name) err := ac.NextComponent().CommitData(options) if err == nil { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - // TODO: Could we just update the size, etag, modtime of the file here? - ac.invalidatePath(options.Name) + ac.lru.invalidatePath(options.Name) } return err } // ------------------------- Factory ------------------------------------------- -// Pipeline will call this method to create your object, initialize your variables here -// << DO NOT DELETE ANY AUTO GENERATED CODE HERE >> +// NewAttrCacheComponent creates a new AttrCache component. func NewAttrCacheComponent() internal.Component { comp := &AttrCache{} comp.SetName(compName) @@ -681,7 +571,6 @@ func NewAttrCacheComponent() internal.Component { return comp } -// On init register this component to pipeline and supply your constructor func init() { internal.AddComponent(compName, NewAttrCacheComponent) @@ -694,4 +583,8 @@ func init() { cacheOnList := config.AddBoolFlag("cache-on-list", true, "Cache attributes on listing.") config.BindPFlag(compName+".cache-on-list", cacheOnList) cacheOnList.Hidden = true + + attrCacheMaxSizeMB := config.AddUint32Flag("attr-cache-max-size-mb", 0, + "maximum memory in MB that attr-cache can use (0 = auto-tune to 1% of total RAM, clamped to [64 MB, 1 GB])") + config.BindPFlag(compName+".max-size-mb", attrCacheMaxSizeMB) } diff --git a/component/attr_cache/attr_cache_integration_test.go b/component/attr_cache/attr_cache_integration_test.go new file mode 100644 index 000000000..2c160f58b --- /dev/null +++ b/component/attr_cache/attr_cache_integration_test.go @@ -0,0 +1,921 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2026 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package attr_cache + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/Azure/azure-storage-fuse/v2/common" + "github.com/Azure/azure-storage-fuse/v2/common/config" + "github.com/Azure/azure-storage-fuse/v2/common/log" + lbfs "github.com/Azure/azure-storage-fuse/v2/component/loopback" + "github.com/Azure/azure-storage-fuse/v2/internal" + "github.com/Azure/azure-storage-fuse/v2/internal/handlemap" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// attrCacheIntegrationTestSuite wires AttrCache -> LoopbackFS so every +// operation actually touches the filesystem. No mocks are used. +type attrCacheIntegrationTestSuite struct { + suite.Suite + assert *assert.Assertions + attrCache *AttrCache + loopbackDir string +} + +func (suite *attrCacheIntegrationTestSuite) SetupTest() { + err := log.SetDefaultLogger("silent", common.LogConfig{}) + suite.Require().NoError(err) + suite.assert = assert.New(suite.T()) + + dir, err := os.MkdirTemp("", "attr_cache_integration_*") + suite.Require().NoError(err) + suite.loopbackDir = dir + + cfg := fmt.Sprintf( + "attr_cache:\n timeout-sec: 30\n max-size-mb: 32\n\nloopbackfs:\n path: %s\n", + dir, + ) + suite.Require().NoError(config.ReadConfigFromReader(strings.NewReader(cfg))) + + loopback := lbfs.NewLoopbackFSComponent() + suite.Require().NoError(loopback.Configure(true)) + suite.Require().NoError(loopback.Start(context.Background())) + + ac := NewAttrCacheComponent() + ac.SetNextComponent(loopback) + suite.Require().NoError(ac.Configure(true)) + suite.Require().NoError(ac.Start(context.Background())) + suite.attrCache = ac.(*AttrCache) +} + +func (suite *attrCacheIntegrationTestSuite) TearDownTest() { + _ = suite.attrCache.Stop() + os.RemoveAll(suite.loopbackDir) +} + +// ---- helpers ---------------------------------------------------------------- + +// touchFile creates a file (and any parent dirs) directly in the loopback dir. +func (suite *attrCacheIntegrationTestSuite) touchFile(relPath string) { + full := filepath.Join(suite.loopbackDir, relPath) + suite.Require().NoError(os.MkdirAll(filepath.Dir(full), 0755)) + f, err := os.Create(full) + suite.Require().NoError(err) + f.Close() +} + +// mkDir creates a directory in the loopback dir. +func (suite *attrCacheIntegrationTestSuite) mkDir(relPath string) { + suite.Require().NoError(os.MkdirAll(filepath.Join(suite.loopbackDir, relPath), 0755)) +} + +// cacheViaGetAttr calls GetAttr through the cache, populating the cache. +// It asserts that the path is absent or invalid before the call, then confirms +// it is cached and valid afterwards. +func (suite *attrCacheIntegrationTestSuite) cacheViaGetAttr(path string) *attrCacheItem { + // Pre-condition: entry must not already be present in the LRU. + _, ok := suite.attrCache.lru.Peek(path) + suite.assert.False(ok, "cacheViaGetAttr: path %q already has an entry in the LRU before GetAttr", path) + + _, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: path}) + suite.assert.NoError(err) + + // Post-condition: entry must now be present. + item, ok := suite.attrCache.lru.Peek(path) + suite.assert.True(ok, "cacheViaGetAttr: path %q not found in LRU after GetAttr", path) + return item +} + +// forceExpire moves the cached item's cachedAt into the past so it is stale. +func (suite *attrCacheIntegrationTestSuite) forceExpire(path string) { + if item, ok := suite.attrCache.lru.Peek(path); ok { + expired := *item + expired.cachedAt = time.Now().Add(-suite.attrCache.cacheTimeout - time.Second) + suite.attrCache.lru.Put(path, &expired) + } +} + +// createFileViaCache creates a file through attr_cache and returns the handle. +func (suite *attrCacheIntegrationTestSuite) createFileViaCache(name string) *handlemap.Handle { + h, err := suite.attrCache.CreateFile(internal.CreateFileOptions{Name: name, Mode: 0644}) + suite.assert.NoError(err) + return h +} + +// ---- GetAttr tests ---------------------------------------------------------- + +func (suite *attrCacheIntegrationTestSuite) TestGetAttrCacheMiss() { + suite.touchFile("miss.txt") + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "miss.txt"}) + suite.assert.NoError(err) + suite.assert.Equal("miss.txt", attr.Path) + + item, ok := suite.attrCache.lru.Peek("miss.txt") + suite.assert.True(ok) + suite.assert.True(item.exists) + suite.assert.NotNil(item.attr) +} + +func (suite *attrCacheIntegrationTestSuite) TestGetAttrCacheHit() { + suite.touchFile("hit.txt") + + // First call: populates cache. + suite.cacheViaGetAttr("hit.txt") + + item, _ := suite.attrCache.lru.Peek("hit.txt") + cachedAt := item.cachedAt + + // Second call: must be served from cache; cachedAt must not change. + _, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "hit.txt"}) + suite.assert.NoError(err) + + item, _ = suite.attrCache.lru.Peek("hit.txt") + suite.assert.Equal(cachedAt, item.cachedAt, "cache hit should not refresh cachedAt") +} + +func (suite *attrCacheIntegrationTestSuite) TestGetAttrTTLExpiry() { + suite.touchFile("ttl.txt") + suite.cacheViaGetAttr("ttl.txt") + + suite.forceExpire("ttl.txt") + + item, _ := suite.attrCache.lru.Peek("ttl.txt") + staleTime := item.cachedAt + + // After TTL expiry the cache should refetch, producing a fresh cachedAt. + _, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "ttl.txt"}) + suite.assert.NoError(err) + + item, _ = suite.attrCache.lru.Peek("ttl.txt") + suite.assert.True(item.cachedAt.After(staleTime), "TTL expiry should refresh the cache entry") +} + +func (suite *attrCacheIntegrationTestSuite) TestGetAttrNegativeCache() { + _, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "ghost.txt"}) + suite.assert.True(os.IsNotExist(err)) + + item, ok := suite.attrCache.lru.Peek("ghost.txt") + suite.assert.True(ok, "negative entry must be cached") + suite.assert.False(item.exists) + suite.assert.Nil(item.attr) +} + +func (suite *attrCacheIntegrationTestSuite) TestGetAttrNegativeTTLExpiry() { + // Populate a negative entry. + _, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "late.txt"}) + suite.assert.True(os.IsNotExist(err)) + + suite.forceExpire("late.txt") + + // Create the file in loopback (behind the cache). + suite.touchFile("late.txt") + + // After TTL expires, the cache should re-check and find the file. + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "late.txt"}) + suite.assert.NoError(err) + suite.assert.Equal("late.txt", attr.Path) + + item, _ := suite.attrCache.lru.Peek("late.txt") + suite.assert.True(item.exists, "negative entry should be replaced by positive one") +} + +func (suite *attrCacheIntegrationTestSuite) TestGetAttrValidEntryNotExpired() { + suite.touchFile("fresh.txt") + item := suite.cacheViaGetAttr("fresh.txt") + originalCachedAt := item.cachedAt + + // Repeated calls within TTL must not re-fetch. + for i := 0; i < 5; i++ { + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "fresh.txt"}) + suite.assert.NoError(err) + suite.assert.Equal("fresh.txt", attr.Path) + } + item, _ = suite.attrCache.lru.Peek("fresh.txt") + suite.assert.Equal(originalCachedAt, item.cachedAt) +} + +// ---- Directory tests -------------------------------------------------------- + +func (suite *attrCacheIntegrationTestSuite) TestCreateDirInvalidatesCache() { + // Pre-populate a stale entry for the dir. + suite.mkDir("newdir") + suite.cacheViaGetAttr("newdir") + + // Remove from disk then re-create via the cache. + os.Remove(filepath.Join(suite.loopbackDir, "newdir")) + + err := suite.attrCache.CreateDir(internal.CreateDirOptions{Name: "newdir", Mode: 0755}) + suite.assert.NoError(err) + + // The created path should not be in the cache (invalidated by CreateDir). + _, ok := suite.attrCache.lru.Peek("newdir") + suite.assert.False(ok) +} + +func (suite *attrCacheIntegrationTestSuite) TestDeleteDirMarksDirectoryNegative() { + suite.mkDir("todelete") + suite.cacheViaGetAttr("todelete") + + err := suite.attrCache.DeleteDir(internal.DeleteDirOptions{Name: "todelete"}) + suite.assert.NoError(err) + + item, ok := suite.attrCache.lru.Peek("todelete") + suite.assert.True(ok) + suite.assert.False(item.exists, "deleted dir must be a negative entry") +} + +func (suite *attrCacheIntegrationTestSuite) TestDeleteDirPrefixIsolation() { + // "a" and "ab" must be independent cache entries. + suite.mkDir("a") + suite.mkDir("ab") + suite.cacheViaGetAttr("a") + suite.cacheViaGetAttr("ab") + + err := suite.attrCache.DeleteDir(internal.DeleteDirOptions{Name: "a"}) + suite.assert.NoError(err) + + aItem, _ := suite.attrCache.lru.Peek("a") + suite.assert.False(aItem.exists, `"a" must be negative`) + + abItem, ok := suite.attrCache.lru.Peek("ab") + suite.assert.True(ok) + suite.assert.True(abItem.exists, `"ab" must remain a positive entry`) +} + +func (suite *attrCacheIntegrationTestSuite) TestDeleteDirMarksChildrenNegative() { + suite.mkDir("parent") + suite.mkDir("parent/child") + suite.touchFile("parent/child/file.txt") + + // Cache all three entries. + suite.cacheViaGetAttr("parent") + suite.cacheViaGetAttr("parent/child") + suite.cacheViaGetAttr("parent/child/file.txt") + + // Remove children first, then the empty parent dir. + os.Remove(filepath.Join(suite.loopbackDir, "parent/child/file.txt")) + os.Remove(filepath.Join(suite.loopbackDir, "parent/child")) + err := suite.attrCache.DeleteDir(internal.DeleteDirOptions{Name: "parent"}) + suite.assert.NoError(err) + + for _, p := range []string{"parent", "parent/child", "parent/child/file.txt"} { + item, ok := suite.attrCache.lru.Peek(p) + suite.assert.True(ok, "entry %s should still be in cache", p) + suite.assert.False(item.exists, "entry %s should be negative after parent dir deletion", p) + } +} + +func (suite *attrCacheIntegrationTestSuite) TestRenameDirMarksSourceNegativeInvalidatesDest() { + suite.mkDir("srcdir") + suite.cacheViaGetAttr("srcdir") + + err := suite.attrCache.RenameDir(internal.RenameDirOptions{Src: "srcdir", Dst: "dstdir"}) + suite.assert.NoError(err) + + // Source must be negative. + srcItem, ok := suite.attrCache.lru.Peek("srcdir") + suite.assert.True(ok) + suite.assert.False(srcItem.exists, "renamed-away dir must be a negative entry") + + // Destination, if cached, must be absent (invalidated by rename). + _, ok = suite.attrCache.lru.Peek("dstdir") + suite.assert.False(ok, "dst should be deleted from cache after rename") +} + +func (suite *attrCacheIntegrationTestSuite) TestSyncDirInvalidatesSubtree() { + suite.mkDir("syncdir") + suite.mkDir("syncdir/sub") + suite.touchFile("syncdir/sub/f.txt") + + suite.cacheViaGetAttr("syncdir") + suite.cacheViaGetAttr("syncdir/sub") + suite.cacheViaGetAttr("syncdir/sub/f.txt") + + err := suite.attrCache.SyncDir(internal.SyncDirOptions{Name: "syncdir"}) + suite.assert.NoError(err) + + for _, p := range []string{"syncdir", "syncdir/sub", "syncdir/sub/f.txt"} { + _, ok := suite.attrCache.lru.Peek(p) + suite.assert.False(ok, "SyncDir must remove %s from cache", p) + } +} + +// ---- ReadDir / StreamDir tests ---------------------------------------------- + +func (suite *attrCacheIntegrationTestSuite) TestReadDirPopulatesCache() { + suite.mkDir("listdir") + suite.touchFile("listdir/a.txt") + suite.touchFile("listdir/b.txt") + + pathList, err := suite.attrCache.ReadDir(internal.ReadDirOptions{Name: "listdir"}) + suite.assert.NoError(err) + suite.assert.Len(pathList, 2) + + for _, attr := range pathList { + item, ok := suite.attrCache.lru.Peek(attr.Path) + suite.assert.True(ok, "ReadDir must cache %s", attr.Path) + suite.assert.True(item.exists) + } +} + +func (suite *attrCacheIntegrationTestSuite) TestReadDirCacheHitOnSubsequentGetAttr() { + suite.mkDir("listdir2") + suite.touchFile("listdir2/c.txt") + + _, err := suite.attrCache.ReadDir(internal.ReadDirOptions{Name: "listdir2"}) + suite.assert.NoError(err) + + item, _ := suite.attrCache.lru.Peek("listdir2/c.txt") + cachedAt := item.cachedAt + + // GetAttr must be served from cache; cachedAt unchanged. + _, err = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "listdir2/c.txt"}) + suite.assert.NoError(err) + + item, _ = suite.attrCache.lru.Peek("listdir2/c.txt") + suite.assert.Equal(cachedAt, item.cachedAt, "GetAttr after ReadDir should be a cache hit") +} + +func (suite *attrCacheIntegrationTestSuite) TestStreamDirPopulatesCache() { + suite.mkDir("streamdir") + suite.touchFile("streamdir/x.txt") + suite.touchFile("streamdir/y.txt") + + pathList, _, err := suite.attrCache.StreamDir(internal.StreamDirOptions{Name: "streamdir"}) + suite.assert.NoError(err) + suite.assert.Len(pathList, 2) + + for _, attr := range pathList { + item, ok := suite.attrCache.lru.Peek(attr.Path) + suite.assert.True(ok, "StreamDir must cache %s", attr.Path) + suite.assert.True(item.exists) + } +} + +func (suite *attrCacheIntegrationTestSuite) TestStreamDirCacheHitOnSubsequentGetAttr() { + suite.mkDir("streamdir2") + suite.touchFile("streamdir2/z.txt") + + _, _, err := suite.attrCache.StreamDir(internal.StreamDirOptions{Name: "streamdir2"}) + suite.assert.NoError(err) + + item, _ := suite.attrCache.lru.Peek("streamdir2/z.txt") + cachedAt := item.cachedAt + + _, err = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "streamdir2/z.txt"}) + suite.assert.NoError(err) + + item, _ = suite.attrCache.lru.Peek("streamdir2/z.txt") + suite.assert.Equal(cachedAt, item.cachedAt, "GetAttr after StreamDir should be a cache hit") +} + +// ---- File operation tests --------------------------------------------------- + +func (suite *attrCacheIntegrationTestSuite) TestCreateFileInvalidatesEntry() { + // Pre-populate a stale entry for a non-existent path. + suite.attrCache.lru.Put("newfile.txt", + &attrCacheItem{cachedAt: time.Now()}) + + h, err := suite.attrCache.CreateFile(internal.CreateFileOptions{Name: "newfile.txt", Mode: 0644}) + suite.assert.NoError(err) + h.GetFileObject().Close() + + _, ok := suite.attrCache.lru.Peek("newfile.txt") + suite.assert.False(ok, "CreateFile must remove the cache entry") +} + +func (suite *attrCacheIntegrationTestSuite) TestDeleteFileMarksEntryNegative() { + suite.touchFile("del.txt") + suite.cacheViaGetAttr("del.txt") + + err := suite.attrCache.DeleteFile(internal.DeleteFileOptions{Name: "del.txt"}) + suite.assert.NoError(err) + + item, ok := suite.attrCache.lru.Peek("del.txt") + suite.assert.True(ok) + suite.assert.False(item.exists, "deleted file must be a negative entry") + + // GetAttr must return ENOENT from cache. + _, err = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "del.txt"}) + suite.assert.Equal(syscall.ENOENT, err) +} + +func (suite *attrCacheIntegrationTestSuite) TestDeleteFileGetAttrAfterTTLReturnsENOENT() { + // Negative entry remains negative even after TTL if file is still gone. + suite.touchFile("gone.txt") + suite.cacheViaGetAttr("gone.txt") + + err := suite.attrCache.DeleteFile(internal.DeleteFileOptions{Name: "gone.txt"}) + suite.assert.NoError(err) + + suite.forceExpire("gone.txt") + + _, err = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "gone.txt"}) + suite.assert.True(os.IsNotExist(err)) +} + +func (suite *attrCacheIntegrationTestSuite) TestRenameFileMarksSourceNegativeUpdatesDest() { + suite.touchFile("src.txt") + srcItem := suite.cacheViaGetAttr("src.txt") + srcAttr := srcItem.attr + + err := suite.attrCache.RenameFile(internal.RenameFileOptions{ + Src: "src.txt", + Dst: "dst.txt", + SrcAttr: srcAttr, + }) + suite.assert.NoError(err) + + srcEntry, ok := suite.attrCache.lru.Peek("src.txt") + suite.assert.True(ok) + suite.assert.False(srcEntry.exists, "renamed-away file must be a negative entry") + + // Destination must be accessible. + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "dst.txt"}) + suite.assert.NoError(err) + suite.assert.Equal("dst.txt", attr.Path) +} + +func (suite *attrCacheIntegrationTestSuite) TestRenameFilePreCachedDestGetsUpdatedAttrs() { + suite.touchFile("rsrc.txt") + suite.touchFile("rdst.txt") + + srcItem := suite.cacheViaGetAttr("rsrc.txt") + srcAttr := srcItem.attr + + // Pre-cache the destination. + suite.cacheViaGetAttr("rdst.txt") + + err := suite.attrCache.RenameFile(internal.RenameFileOptions{ + Src: "rsrc.txt", + Dst: "rdst.txt", + SrcAttr: srcAttr, + }) + suite.assert.NoError(err) + + dstItem, ok := suite.attrCache.lru.Peek("rdst.txt") + suite.assert.True(ok) + suite.assert.True(dstItem.exists) +} + +func (suite *attrCacheIntegrationTestSuite) TestWriteFileInvalidatesEntry() { + h := suite.createFileViaCache("write.txt") + defer h.GetFileObject().Close() + + suite.cacheViaGetAttr("write.txt") + + _, err := suite.attrCache.WriteFile(&internal.WriteFileOptions{ + Handle: h, + Offset: 0, + Data: []byte("hello world"), + }) + suite.assert.NoError(err) + + _, ok := suite.attrCache.lru.Peek("write.txt") + suite.assert.False(ok, "WriteFile must remove the cache entry") +} + +func (suite *attrCacheIntegrationTestSuite) TestTruncateFileInvalidatesEntry() { + suite.touchFile("trunc.txt") + suite.cacheViaGetAttr("trunc.txt") + + err := suite.attrCache.TruncateFile(internal.TruncateFileOptions{ + Name: "trunc.txt", + NewSize: 0, + }) + suite.assert.NoError(err) + + _, ok := suite.attrCache.lru.Peek("trunc.txt") + suite.assert.False(ok, "TruncateFile must remove the cache entry") +} + +func (suite *attrCacheIntegrationTestSuite) TestFlushFileInvalidatesEntry() { + h := suite.createFileViaCache("flush.txt") + defer h.GetFileObject().Close() + + suite.cacheViaGetAttr("flush.txt") + + err := suite.attrCache.FlushFile(internal.FlushFileOptions{Handle: h}) + suite.assert.NoError(err) + + _, ok := suite.attrCache.lru.Peek("flush.txt") + suite.assert.False(ok, "FlushFile must remove the cache entry") +} + +func (suite *attrCacheIntegrationTestSuite) TestSyncFileInvalidatesEntry() { + suite.touchFile("sync.txt") + suite.cacheViaGetAttr("sync.txt") + + h := handlemap.NewHandle("sync.txt") + err := suite.attrCache.SyncFile(internal.SyncFileOptions{Handle: h}) + suite.assert.NoError(err) + + _, ok := suite.attrCache.lru.Peek("sync.txt") + suite.assert.False(ok, "SyncFile must remove the cache entry") +} + +func (suite *attrCacheIntegrationTestSuite) TestChmodUpdatesInPlace() { + suite.touchFile("chmod.txt") + suite.cacheViaGetAttr("chmod.txt") + + newMode := os.FileMode(0600) + err := suite.attrCache.Chmod(internal.ChmodOptions{Name: "chmod.txt", Mode: newMode}) + suite.assert.NoError(err) + + // Chmod must update the mode in-place, not invalidate. + item, ok := suite.attrCache.lru.Peek("chmod.txt") + suite.assert.True(ok) + suite.assert.True(item.exists) + suite.assert.Equal(newMode, item.attr.Mode) +} + +func (suite *attrCacheIntegrationTestSuite) TestChmodOnNonCachedEntryNoOp() { + suite.touchFile("chmod_nc.txt") + // Don't cache it first. + + err := suite.attrCache.Chmod(internal.ChmodOptions{Name: "chmod_nc.txt", Mode: 0600}) + suite.assert.NoError(err) + + // Entry should not be in cache. + _, ok := suite.attrCache.lru.Peek("chmod_nc.txt") + suite.assert.False(ok) +} + +func (suite *attrCacheIntegrationTestSuite) TestCopyFromFileInvalidatesEntry() { + suite.touchFile("copydst.txt") + suite.cacheViaGetAttr("copydst.txt") + + tmpSrc, err := os.CreateTemp("", "copy_src_*") + suite.Require().NoError(err) + defer func() { + tmpSrc.Close() + os.Remove(tmpSrc.Name()) + }() + _, _ = tmpSrc.WriteString("content") + _, _ = tmpSrc.Seek(0, 0) + + err = suite.attrCache.CopyFromFile(internal.CopyFromFileOptions{ + Name: "copydst.txt", + File: tmpSrc, + }) + suite.assert.NoError(err) + + _, ok := suite.attrCache.lru.Peek("copydst.txt") + suite.assert.False(ok, "CopyFromFile must remove the cache entry") +} + +func (suite *attrCacheIntegrationTestSuite) TestCreateLinkInvalidatesBothEnds() { + suite.touchFile("linktarget.txt") + suite.cacheViaGetAttr("linktarget.txt") + + // Pre-populate the link name in cache (as negative so it is present). + suite.attrCache.lru.cacheNegativeEntry("linkname") + + err := suite.attrCache.CreateLink(internal.CreateLinkOptions{ + Name: "linkname", + Target: "linktarget.txt", + }) + suite.assert.NoError(err) + + for _, p := range []string{"linkname", "linktarget.txt"} { + _, ok := suite.attrCache.lru.Peek(p) + suite.assert.False(ok, "CreateLink must remove %s from cache", p) + } +} + +func (suite *attrCacheIntegrationTestSuite) TestCommitDataInvalidatesEntry() { + suite.touchFile("commit.txt") + suite.cacheViaGetAttr("commit.txt") + + err := suite.attrCache.CommitData(internal.CommitDataOptions{ + Name: "commit.txt", + List: []string{}, + }) + suite.assert.NoError(err) + + _, ok := suite.attrCache.lru.Peek("commit.txt") + suite.assert.False(ok, "CommitData must remove the cache entry") +} + +// ---- LRU eviction tests ----------------------------------------------------- + +func (suite *attrCacheIntegrationTestSuite) TestLRUEvictionForcesRefetch() { + for _, name := range []string{"lru_a.txt", "lru_b.txt", "lru_c.txt"} { + suite.touchFile(name) + } + + // Measure a single entry's cost, then restart the cache capped at two entries. + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "lru_a.txt"}) + singleSize := suite.attrCache.lru.Size() + suite.assert.Positive(singleSize) + suite.attrCache.maxSizeBytes = singleSize * 2 + suite.Require().NoError(suite.attrCache.Stop()) + suite.Require().NoError(suite.attrCache.Start(context.Background())) + + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "lru_a.txt"}) + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "lru_b.txt"}) + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "lru_c.txt"}) + + // LRU must have evicted the oldest entry (lru_a). + suite.assert.LessOrEqual(suite.attrCache.lru.Size(), singleSize*2) + suite.assert.LessOrEqual(suite.attrCache.lru.Len(), 2) + + // GetAttr on an evicted entry must still succeed (re-fetched from loopback). + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "lru_a.txt"}) + suite.assert.NoError(err) + suite.assert.Equal("lru_a.txt", attr.Path) +} + +func (suite *attrCacheIntegrationTestSuite) TestLRUEvictionPreservesRecentlyAccessed() { + for _, name := range []string{"ev_a.txt", "ev_b.txt", "ev_c.txt"} { + suite.touchFile(name) + } + + // Measure a single entry's cost, then restart the cache capped at two entries. + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "ev_a.txt"}) + singleSize := suite.attrCache.lru.Size() + suite.attrCache.maxSizeBytes = singleSize * 2 + suite.Require().NoError(suite.attrCache.Stop()) + suite.Require().NoError(suite.attrCache.Start(context.Background())) + + // Populate A and B. + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "ev_a.txt"}) + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "ev_b.txt"}) + + // Promote A to MRU by re-accessing it. + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "ev_a.txt"}) + + // Adding C should evict B (LRU). + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "ev_c.txt"}) + + suite.assert.True(suite.attrCache.lru.Has("ev_a.txt"), "recently accessed A should be retained") + suite.assert.True(suite.attrCache.lru.Has("ev_c.txt"), "newest C should be retained") + suite.assert.False(suite.attrCache.lru.Has("ev_b.txt"), "LRU entry B should have been evicted") +} + +// ---- Concurrency tests ------------------------------------------------------ + +func (suite *attrCacheIntegrationTestSuite) TestConcurrentGetAttrNoRace() { + const n = 10 + for i := 0; i < n; i++ { + suite.touchFile(fmt.Sprintf("conc_%d.txt", i)) + } + + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + name := fmt.Sprintf("conc_%d.txt", idx) + for j := 0; j < 20; j++ { + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: name}) + } + }(i) + } + wg.Wait() + + // Verify all entries are cached correctly after concurrent access. + for i := 0; i < n; i++ { + _, ok := suite.attrCache.lru.Peek(fmt.Sprintf("conc_%d.txt", i)) + suite.assert.True(ok) + } +} + +func (suite *attrCacheIntegrationTestSuite) TestConcurrentMutationsAndReads() { + const n = 5 + for i := 0; i < n; i++ { + suite.touchFile(fmt.Sprintf("mut_%d.txt", i)) + } + + var wg sync.WaitGroup + + // Readers: repeatedly call GetAttr. + for i := 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + for j := 0; j < 30; j++ { + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{ + Name: fmt.Sprintf("mut_%d.txt", idx), + }) + } + }(i) + } + + // Writers: invalidate entries via Chmod. + for i := 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + for j := 0; j < 10; j++ { + _ = suite.attrCache.Chmod(internal.ChmodOptions{ + Name: fmt.Sprintf("mut_%d.txt", idx), + Mode: os.FileMode(0644), + }) + } + }(i) + } + + wg.Wait() +} + +func (suite *attrCacheIntegrationTestSuite) TestConcurrentGetAttrAndNilAttrRace() { + // This test specifically exercises the crash that occurred before items became immutable: + // + // Thread A: lru.Get("f") → returns *item with valid=true, exists=true, attr= + // Thread B: invalidatePath("f") → used to mutate item.attr=nil in-place + // Thread A: item.getAttr() → returned nil → caller dereferences → crash + // + // With immutable items, Thread B's Put replaces the LRU entry with a NEW item; Thread A + // still holds the OLD pointer whose attr field is never mutated, so no nil dereference. + const n = 8 + for i := 0; i < n; i++ { + suite.touchFile(fmt.Sprintf("race_%d.txt", i)) + } + + // Pre-populate cache so readers find valid entries immediately. + for i := 0; i < n; i++ { + _, _ = suite.attrCache.GetAttr(internal.GetAttrOptions{Name: fmt.Sprintf("race_%d.txt", i)}) + } + + var wg sync.WaitGroup + + // Readers: call GetAttr and assert the returned attr is never nil. + for i := 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + name := fmt.Sprintf("race_%d.txt", idx) + for j := 0; j < 50; j++ { + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: name}) + if err == nil { + // A nil attr on a successful GetAttr is the crash scenario. + suite.assert.NotNil(attr, "GetAttr must never return nil attr on success") + } + } + }(i) + } + + // Invalidators: repeatedly invalidate the same entries via DeleteFile + re-create. + for i := 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + name := fmt.Sprintf("race_%d.txt", idx) + for j := 0; j < 20; j++ { + _ = suite.attrCache.DeleteFile(internal.DeleteFileOptions{Name: name}) + full := filepath.Join(suite.loopbackDir, name) + _ = os.MkdirAll(filepath.Dir(full), 0755) + if f, err := os.Create(full); err == nil { + f.Close() + } + h, err := suite.attrCache.CreateFile(internal.CreateFileOptions{Name: name, Mode: 0644}) + if err == nil { + h.GetFileObject().Close() + } + } + }(i) + } + + // Directory invalidators: exercise the two-phase directory invalidation path. + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 10; j++ { + _ = suite.attrCache.SyncDir(internal.SyncDirOptions{Name: ""}) + } + }() + + wg.Wait() +} + +func (suite *attrCacheIntegrationTestSuite) TestGetAttrOnRootDirectory() { + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: ""}) + // Root directory should resolve to the loopback path — either a valid attr or ENOENT. + if err == nil { + suite.assert.NotNil(attr) + } else { + suite.assert.True(os.IsNotExist(err)) + } +} + +func (suite *attrCacheIntegrationTestSuite) TestCacheEntriesDoNotCrossContainerBoundary() { + // "dir" and "dir2" must have independent entries even though one is a prefix of the other. + suite.mkDir("dir") + suite.mkDir("dir2") + suite.touchFile("dir/f.txt") + suite.touchFile("dir2/f.txt") + + suite.cacheViaGetAttr("dir") + suite.cacheViaGetAttr("dir2") + suite.cacheViaGetAttr("dir/f.txt") + suite.cacheViaGetAttr("dir2/f.txt") + + // Remove the child file and child dir from disk so the parent dir is empty + // for DeleteDir (loopback uses os.Remove which requires an empty directory). + os.Remove(filepath.Join(suite.loopbackDir, "dir/f.txt")) + + err := suite.attrCache.DeleteDir(internal.DeleteDirOptions{Name: "dir"}) + suite.assert.NoError(err) + + dirItem, _ := suite.attrCache.lru.Peek("dir") + suite.assert.False(dirItem.exists, `"dir" must be negative`) + + dir2Item, _ := suite.attrCache.lru.Peek("dir2") + suite.assert.True(dir2Item.exists, `"dir2" must remain positive`) + + dirFItem, _ := suite.attrCache.lru.Peek("dir/f.txt") + suite.assert.False(dirFItem.exists, `"dir/f.txt" must be negative`) + + dir2FItem, _ := suite.attrCache.lru.Peek("dir2/f.txt") + suite.assert.True(dir2FItem.exists, `"dir2/f.txt" must remain positive`) +} + +func (suite *attrCacheIntegrationTestSuite) TestStaleEntryReplacedOnRefetch() { + suite.touchFile("stale.txt") + suite.cacheViaGetAttr("stale.txt") + + // Modify the file directly in the backing store. + err := os.WriteFile(filepath.Join(suite.loopbackDir, "stale.txt"), []byte("new content"), 0644) + suite.assert.NoError(err) + + // Entry is still valid (not expired), so GetAttr returns cached attrs. + suite.forceExpire("stale.txt") + + // After expiry, the new attributes should be fetched. + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "stale.txt"}) + suite.assert.NoError(err) + suite.assert.Positive(attr.Size, "refreshed entry must reflect new file size") +} + +func (suite *attrCacheIntegrationTestSuite) TestDeleteThenRecreateFile() { + suite.touchFile("lifecycle.txt") + suite.cacheViaGetAttr("lifecycle.txt") + + _ = suite.attrCache.DeleteFile(internal.DeleteFileOptions{Name: "lifecycle.txt"}) + + // Verify negative entry. + item, _ := suite.attrCache.lru.Peek("lifecycle.txt") + suite.assert.False(item.exists) + + // Expire the negative entry. + suite.forceExpire("lifecycle.txt") + + // Recreate via attr_cache. + h, err := suite.attrCache.CreateFile(internal.CreateFileOptions{Name: "lifecycle.txt", Mode: 0644}) + suite.assert.NoError(err) + h.GetFileObject().Close() + + // GetAttr must now succeed. + attr, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: "lifecycle.txt"}) + suite.assert.NoError(err) + suite.assert.Equal("lifecycle.txt", attr.Path) +} + +// ---- Test runner ------------------------------------------------------------ + +func TestAttrCacheIntegration(t *testing.T) { + suite.Run(t, new(attrCacheIntegrationTestSuite)) +} diff --git a/component/attr_cache/attr_cache_test.go b/component/attr_cache/attr_cache_test.go index 8d6e71abc..fdd65c779 100644 --- a/component/attr_cache/attr_cache_test.go +++ b/component/attr_cache/attr_cache_test.go @@ -94,44 +94,53 @@ func getPathAttr(path string, size int64, mode os.FileMode, metadata bool) *inte } } +// getCacheItem returns the cached item for path without altering LRU order. +func getCacheItem(ac *AttrCache, path string) *attrCacheItem { + item, ok := ac.lru.Peek(path) + if !ok { + return nil + } + return item +} + func addPathToCache(assert *assert.Assertions, attrCache *AttrCache, path string, metadata bool) { path = internal.TruncateDirName(path) - attrCache.cacheMap[path] = newAttrCacheItem(getPathAttr(path, defaultSize, fs.FileMode(defaultMode), metadata), true, time.Now()) - assert.Contains(attrCache.cacheMap, path) + item := &attrCacheItem{attr: getPathAttr(path, defaultSize, fs.FileMode(defaultMode), metadata), exists: true, cachedAt: time.Now()} + attrCache.lru.Put(path, item) + assert.True(attrCache.lru.Has(path)) } func assertDeleted(suite *attrCacheTestSuite, path string) { - suite.assert.Contains(suite.attrCache.cacheMap, path) - suite.assert.Equal(&internal.ObjAttr{}, suite.attrCache.cacheMap[path].attr) - suite.assert.True(suite.attrCache.cacheMap[path].valid()) - suite.assert.False(suite.attrCache.cacheMap[path].exists()) + item := getCacheItem(suite.attrCache, path) + suite.assert.NotNil(item) + suite.assert.Nil(item.attr) + suite.assert.False(item.exists) } func assertInvalid(suite *attrCacheTestSuite, path string) { - suite.assert.Contains(suite.attrCache.cacheMap, path) - suite.assert.Equal(&internal.ObjAttr{}, suite.attrCache.cacheMap[path].attr) - suite.assert.False(suite.attrCache.cacheMap[path].valid()) + item := getCacheItem(suite.attrCache, path) + suite.assert.Nil(item) } func assertUntouched(suite *attrCacheTestSuite, path string) { - suite.assert.Contains(suite.attrCache.cacheMap, path) - suite.assert.NotEqual(&internal.ObjAttr{}, suite.attrCache.cacheMap[path].attr) - suite.assert.Equal(suite.attrCache.cacheMap[path].attr.Size, defaultSize) - suite.assert.EqualValues(suite.attrCache.cacheMap[path].attr.Mode, defaultMode) - suite.assert.True(suite.attrCache.cacheMap[path].valid()) - suite.assert.True(suite.attrCache.cacheMap[path].exists()) + item := getCacheItem(suite.attrCache, path) + suite.assert.NotNil(item) + suite.assert.NotEqual(&internal.ObjAttr{}, item.attr) + suite.assert.Equal(item.attr.Size, defaultSize) + suite.assert.EqualValues(item.attr.Mode, defaultMode) + suite.assert.True(item.exists) } -// This method is used when we transfer the attributes from the src to dst, and mark src as invalid +// assertAttributesTransferred checks that dst has the same attrs as src and is valid/exists. func assertAttributesTransferred(suite *attrCacheTestSuite, srcAttr *internal.ObjAttr, dstAttr *internal.ObjAttr) { suite.assert.Equal(srcAttr.Size, dstAttr.Size) - suite.assert.Equal(srcAttr.Path, dstAttr.Path) suite.assert.Equal(srcAttr.Mode, dstAttr.Mode) suite.assert.Equal(srcAttr.Atime, dstAttr.Atime) suite.assert.Equal(srcAttr.Mtime, dstAttr.Mtime) suite.assert.Equal(srcAttr.Ctime, dstAttr.Ctime) - suite.assert.True(suite.attrCache.cacheMap[dstAttr.Path].exists()) - suite.assert.True(suite.attrCache.cacheMap[dstAttr.Path].valid()) + dstItem := getCacheItem(suite.attrCache, dstAttr.Path) + suite.assert.NotNil(dstItem) + suite.assert.True(dstItem.exists) } // If next component changes the times of the attribute. @@ -225,7 +234,7 @@ func (suite *attrCacheTestSuite) cleanupTest() { func (suite *attrCacheTestSuite) TestDefault() { defer suite.cleanupTest() suite.assert.Equal("attr_cache", suite.attrCache.Name()) - suite.assert.EqualValues(120, suite.attrCache.cacheTimeout) + suite.assert.Equal(120*time.Second, suite.attrCache.cacheTimeout) // suite.assert.Equal(suite.attrCache.noSymlinks, false) } @@ -237,19 +246,19 @@ func (suite *attrCacheTestSuite) TestConfig() { suite.setupTestHelper(config) // setup a new attr cache with a custom config (clean up will occur after the test as usual) suite.assert.Equal("attr_cache", suite.attrCache.Name()) - suite.assert.EqualValues(60, suite.attrCache.cacheTimeout) + suite.assert.Equal(60*time.Second, suite.attrCache.cacheTimeout) suite.assert.True(suite.attrCache.noSymlinks) } -// Tests max files config -func (suite *attrCacheTestSuite) TestConfigMaxFiles() { +// Tests max-size-mb config +func (suite *attrCacheTestSuite) TestConfigMaxSizeMB() { defer suite.cleanupTest() suite.cleanupTest() // clean up the default attr cache generated cacheTimeout := 1 - maxFiles := 10 - config := fmt.Sprintf("attr_cache:\n timeout-sec: %d\n max-files: %d", cacheTimeout, maxFiles) + maxSizeMB := 256 + config := fmt.Sprintf("attr_cache:\n timeout-sec: %d\n max-size-mb: %d", cacheTimeout, maxSizeMB) suite.setupTestHelper(config) // setup a new attr cache with a custom config (clean up will occur after the test as usual) - suite.assert.Equal(suite.attrCache.maxFiles, maxFiles) + suite.assert.Equal(suite.attrCache.maxSizeBytes, int64(maxSizeMB)*1024*1024) } func (suite *attrCacheTestSuite) TestConfigZero() { @@ -282,7 +291,7 @@ func (suite *attrCacheTestSuite) TestCreateDir() { err := suite.attrCache.CreateDir(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Success // Entry Does Not Already Exist @@ -290,7 +299,7 @@ func (suite *attrCacheTestSuite) TestCreateDir() { err = suite.attrCache.CreateDir(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Entry Already Exists addPathToCache(suite.assert, suite.attrCache, path, false) @@ -321,7 +330,7 @@ func (suite *attrCacheTestSuite) TestDeleteDir() { err := suite.attrCache.DeleteDir(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Success // Entry Does Not Already Exist @@ -329,7 +338,7 @@ func (suite *attrCacheTestSuite) TestDeleteDir() { err = suite.attrCache.DeleteDir(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + assertDeleted(suite, truncatedPath) // Entry Already Exists a, ab, ac := addDirectoryToCache(suite.assert, suite.attrCache, path, false) @@ -372,20 +381,20 @@ func (suite *attrCacheTestSuite) TestReadDirDoesNotExist() { // Entries Do Not Already Exist suite.mock.EXPECT().ReadDir(options).Return(aAttr, nil) - suite.assert.Empty(suite.attrCache.cacheMap) // cacheMap should be empty before call + suite.assert.Zero(suite.attrCache.lru.Len()) // cache should be empty before call returnedAttr, err := suite.attrCache.ReadDir(options) suite.assert.NoError(err) suite.assert.Equal(aAttr, returnedAttr) - suite.assert.Len(aAttr, len(suite.attrCache.cacheMap)) + suite.assert.Len(aAttr, suite.attrCache.lru.Len()) // Entries should now be in the cache for _, p := range aAttr { - suite.assert.Contains(suite.attrCache.cacheMap, p.Path) - suite.assert.NotEqual(&internal.ObjAttr{}, suite.attrCache.cacheMap[p.Path].attr) - suite.assert.Equal(suite.attrCache.cacheMap[p.Path].attr.Size, size) // new size should be set - suite.assert.Equal(suite.attrCache.cacheMap[p.Path].attr.Mode, mode) // new mode should be set - suite.assert.True(suite.attrCache.cacheMap[p.Path].valid()) - suite.assert.True(suite.attrCache.cacheMap[p.Path].exists()) + item := getCacheItem(suite.attrCache, p.Path) + suite.assert.NotNil(item) + suite.assert.NotEqual(&internal.ObjAttr{}, item.attr) + suite.assert.Equal(item.attr.Size, size) // new size should be set + suite.assert.Equal(item.attr.Mode, mode) // new mode should be set + suite.assert.True(item.exists) } }) } @@ -410,7 +419,7 @@ func (suite *attrCacheTestSuite) TestReadDirExists() { // Entries Already Exist a, ab, ac := addDirectoryToCache(suite.assert, suite.attrCache, path, false) - suite.assert.NotEmpty(suite.attrCache.cacheMap) // cacheMap should NOT be empty before read dir call and values should be untouched + suite.assert.NotZero(suite.attrCache.lru.Len()) // cache should NOT be empty before read dir call and values should be untouched for _, p := range aAttr { assertUntouched(suite, p.Path) } @@ -422,12 +431,12 @@ func (suite *attrCacheTestSuite) TestReadDirExists() { // a paths should now be updated in the cache for p := a.Front(); p != nil; p = p.Next() { pString := p.Value.(string) - suite.assert.Contains(suite.attrCache.cacheMap, pString) - suite.assert.NotEqual(&internal.ObjAttr{}, suite.attrCache.cacheMap[pString].attr) - suite.assert.Equal(suite.attrCache.cacheMap[pString].attr.Size, size) // new size should be set - suite.assert.Equal(suite.attrCache.cacheMap[pString].attr.Mode, mode) // new mode should be set - suite.assert.True(suite.attrCache.cacheMap[pString].valid()) - suite.assert.True(suite.attrCache.cacheMap[pString].exists()) + item := getCacheItem(suite.attrCache, pString) + suite.assert.NotNil(item) + suite.assert.NotEqual(&internal.ObjAttr{}, item.attr) + suite.assert.Equal(item.attr.Size, size) // new size should be set + suite.assert.Equal(item.attr.Mode, mode) // new mode should be set + suite.assert.True(item.exists) } // ab and ac paths should be untouched @@ -456,7 +465,7 @@ func (suite *attrCacheTestSuite) TestReadDirError() { _, err := suite.attrCache.ReadDir(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) }) } } @@ -488,8 +497,8 @@ func (suite *attrCacheTestSuite) TestRenameDir() { err := suite.attrCache.RenameDir(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedSrc) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedDst) + suite.assert.False(suite.attrCache.lru.Has(truncatedSrc)) + suite.assert.False(suite.attrCache.lru.Has(truncatedDst)) // Success // Entry Does Not Already Exist @@ -497,8 +506,8 @@ func (suite *attrCacheTestSuite) TestRenameDir() { err = suite.attrCache.RenameDir(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedSrc) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedDst) + assertDeleted(suite, truncatedSrc) + suite.assert.False(suite.attrCache.lru.Has(truncatedDst)) // Entry Already Exists a, ab, ac := addDirectoryToCache(suite.assert, suite.attrCache, input.src, false) @@ -538,7 +547,7 @@ func (suite *attrCacheTestSuite) TestCreateFile() { _, err := suite.attrCache.CreateFile(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(path)) // Success // Entry Does Not Already Exist @@ -546,7 +555,7 @@ func (suite *attrCacheTestSuite) TestCreateFile() { _, err = suite.attrCache.CreateFile(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(path)) // Entry Already Exists addPathToCache(suite.assert, suite.attrCache, path, false) @@ -569,7 +578,7 @@ func (suite *attrCacheTestSuite) TestDeleteFile() { err := suite.attrCache.DeleteFile(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(path)) // Success // Entry Does Not Already Exist @@ -577,7 +586,7 @@ func (suite *attrCacheTestSuite) TestDeleteFile() { err = suite.attrCache.DeleteFile(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + assertDeleted(suite, path) // Entry Already Exists addPathToCache(suite.assert, suite.attrCache, path, false) @@ -603,7 +612,7 @@ func (suite *attrCacheTestSuite) TestSyncFile() { err := suite.attrCache.SyncFile(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(path)) // Success // Entry Does Not Already Exist @@ -611,7 +620,7 @@ func (suite *attrCacheTestSuite) TestSyncFile() { err = suite.attrCache.SyncFile(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(path)) // Entry Already Exists addPathToCache(suite.assert, suite.attrCache, path, false) @@ -639,7 +648,7 @@ func (suite *attrCacheTestSuite) TestSyncDir() { err := suite.attrCache.SyncDir(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Success // Entry Does Not Already Exist @@ -647,7 +656,7 @@ func (suite *attrCacheTestSuite) TestSyncDir() { err = suite.attrCache.SyncDir(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Entry Already Exists a, ab, ac := addDirectoryToCache(suite.assert, suite.attrCache, path, false) @@ -683,8 +692,8 @@ func (suite *attrCacheTestSuite) TestRenameFile() { err := suite.attrCache.RenameFile(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, src) - suite.assert.NotContains(suite.attrCache.cacheMap, dst) + suite.assert.False(suite.attrCache.lru.Has(src)) + suite.assert.False(suite.attrCache.lru.Has(dst)) // Success // Entry Does Not Already Exist @@ -692,16 +701,16 @@ func (suite *attrCacheTestSuite) TestRenameFile() { err = suite.attrCache.RenameFile(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, src) - suite.assert.NotContains(suite.attrCache.cacheMap, dst) + assertDeleted(suite, src) + suite.assert.False(suite.attrCache.lru.Has(dst)) // Src, Dst Entry Already Exists addPathToCache(suite.assert, suite.attrCache, src, false) addPathToCache(suite.assert, suite.attrCache, dst, false) - options.SrcAttr = suite.attrCache.cacheMap[src].attr + options.SrcAttr = getCacheItem(suite.attrCache, src).attr options.SrcAttr.Size = 1 options.SrcAttr.Mode = 2 - options.DstAttr = suite.attrCache.cacheMap[dst].attr + options.DstAttr = getCacheItem(suite.attrCache, dst).attr options.DstAttr.Size = 3 options.DstAttr.Mode = 4 srcAttrCopy := *options.SrcAttr @@ -710,7 +719,7 @@ func (suite *attrCacheTestSuite) TestRenameFile() { err = suite.attrCache.RenameFile(options) suite.assert.NoError(err) assertDeleted(suite, src) - modifiedDstAttr := suite.attrCache.cacheMap[dst].attr + modifiedDstAttr := getCacheItem(suite.attrCache, dst).attr assertSrcAttributeTimeChanged(suite, options.SrcAttr, srcAttrCopy) // Check the attributes of the dst are same as the src. assertAttributesTransferred(suite, options.SrcAttr, modifiedDstAttr) @@ -718,16 +727,16 @@ func (suite *attrCacheTestSuite) TestRenameFile() { // Src Entry Exist and Dst Entry Don't Exist addPathToCache(suite.assert, suite.attrCache, src, false) // Add negative entry to cache for Dst - suite.attrCache.cacheMap[dst] = newAttrCacheItem(&internal.ObjAttr{}, false, time.Now()) - options.SrcAttr = suite.attrCache.cacheMap[src].attr - options.DstAttr = suite.attrCache.cacheMap[dst].attr + suite.attrCache.lru.Put(dst, &attrCacheItem{cachedAt: time.Now()}) + options.SrcAttr = getCacheItem(suite.attrCache, src).attr + options.DstAttr = getCacheItem(suite.attrCache, dst).attr options.SrcAttr.Size = 1 options.SrcAttr.Mode = 2 suite.mock.EXPECT().RenameFile(options).Return(nil) err = suite.attrCache.RenameFile(options) suite.assert.NoError(err) assertDeleted(suite, src) - modifiedDstAttr = suite.attrCache.cacheMap[dst].attr + modifiedDstAttr = getCacheItem(suite.attrCache, dst).attr assertSrcAttributeTimeChanged(suite, options.SrcAttr, srcAttrCopy) assertAttributesTransferred(suite, options.SrcAttr, modifiedDstAttr) } @@ -748,7 +757,7 @@ func (suite *attrCacheTestSuite) TestWriteFileError() { _, err := suite.attrCache.WriteFile(&options) suite.assert.Error(err) - suite.assert.Contains(suite.attrCache.cacheMap, path) // GetAttr call will add this to the cache + suite.assert.True(suite.attrCache.lru.Has(path)) // GetAttr call will add this to the cache } func (suite *attrCacheTestSuite) TestWriteFileDoesNotExist() { @@ -766,7 +775,7 @@ func (suite *attrCacheTestSuite) TestWriteFileDoesNotExist() { _, err := suite.attrCache.WriteFile(&options) suite.assert.NoError(err) - suite.assert.Contains(suite.attrCache.cacheMap, path) // GetAttr call will add this to the cache + suite.assert.False(suite.attrCache.lru.Has(path)) // invalidated by WriteFile } func (suite *attrCacheTestSuite) TestWriteFileExists() { @@ -799,7 +808,7 @@ func (suite *attrCacheTestSuite) TestTruncateFile() { err := suite.attrCache.TruncateFile(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(path)) // Success // Entry Does Not Already Exist @@ -807,7 +816,7 @@ func (suite *attrCacheTestSuite) TestTruncateFile() { err = suite.attrCache.TruncateFile(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(path)) // Entry Already Exists addPathToCache(suite.assert, suite.attrCache, path, false) @@ -815,13 +824,8 @@ func (suite *attrCacheTestSuite) TestTruncateFile() { err = suite.attrCache.TruncateFile(options) suite.assert.NoError(err) - // suite.assert.Contains(suite.attrCache.cacheMap, path) - // suite.assert.NotEqualValues(suite.attrCache.cacheMap[path].attr, &internal.ObjAttr{}) - // suite.assert.EqualValues(suite.attrCache.cacheMap[path].attr.Size, size) // new size should be set - // suite.assert.EqualValues(suite.attrCache.cacheMap[path].attr.Mode, defaultMode) - // suite.assert.True(suite.attrCache.cacheMap[path].valid()) - // suite.assert.True(suite.attrCache.cacheMap[path].exists()) - suite.assert.False(suite.attrCache.cacheMap[path].valid()) + item := getCacheItem(suite.attrCache, path) + suite.assert.Nil(item) } // Tests CopyFromFile @@ -836,7 +840,7 @@ func (suite *attrCacheTestSuite) TestCopyFromFileError() { err := suite.attrCache.CopyFromFile(options) suite.assert.Error(err) - suite.assert.Contains(suite.attrCache.cacheMap, path) // GetAttr call will add this to the cache + suite.assert.True(suite.attrCache.lru.Has(path)) // GetAttr call will add this to the cache } func (suite *attrCacheTestSuite) TestCopyFromFileDoesNotExist() { @@ -851,7 +855,7 @@ func (suite *attrCacheTestSuite) TestCopyFromFileDoesNotExist() { err := suite.attrCache.CopyFromFile(options) suite.assert.NoError(err) - suite.assert.Contains(suite.attrCache.cacheMap, path) // GetAttr call will add this to the cache + suite.assert.False(suite.attrCache.lru.Has(path)) // invalidated by CopyFromFile } func (suite *attrCacheTestSuite) TestCopyFromFileExists() { @@ -955,8 +959,6 @@ func (suite *attrCacheTestSuite) TestGetAttrExistsWithoutMetadata() { addDirectoryToCache(suite.assert, suite.attrCache, "a", false) // add the paths to the cache with IsMetadataRetrived=false options := internal.GetAttrOptions{Name: path} - // attributes should not be accessible so call the mock - //suite.mock.EXPECT().GetAttr(options).Return(getPathAttr(path, defaultSize, fs.FileMode(defaultMode), false), nil) _, err := suite.attrCache.GetAttr(options) suite.assert.NoError(err) @@ -980,7 +982,7 @@ func (suite *attrCacheTestSuite) TestGetAttrDoesNotExist() { // attributes should not be accessible so call the mock suite.mock.EXPECT().GetAttr(options).Return(getPathAttr(path, defaultSize, fs.FileMode(defaultMode), false), nil) - suite.assert.Empty(suite.attrCache.cacheMap) // cacheMap should be empty before call + suite.assert.Zero(suite.attrCache.lru.Len()) // cache should be empty before call _, err := suite.attrCache.GetAttr(options) suite.assert.NoError(err) assertUntouched(suite, truncatedPath) // item added to cache after @@ -999,13 +1001,14 @@ func (suite *attrCacheTestSuite) TestGetAttrOtherError() { suite.Run(path, func() { truncatedPath := internal.TruncateDirName(path) + someErr := errors.New("some unexpected error") options := internal.GetAttrOptions{Name: path} - suite.mock.EXPECT().GetAttr(options).Return(&internal.ObjAttr{}, os.ErrNotExist) + suite.mock.EXPECT().GetAttr(options).Return(&internal.ObjAttr{}, someErr) result, err := suite.attrCache.GetAttr(options) - suite.assert.Equal(err, os.ErrNotExist) + suite.assert.Equal(err, someErr) suite.assert.Equal(&internal.ObjAttr{}, result) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) }) } } @@ -1027,15 +1030,55 @@ func (suite *attrCacheTestSuite) TestGetAttrEnonetError() { result, err := suite.attrCache.GetAttr(options) suite.assert.Equal(syscall.ENOENT, err) suite.assert.Equal(&internal.ObjAttr{}, result) - suite.assert.Contains(suite.attrCache.cacheMap, truncatedPath) - suite.assert.Equal(&internal.ObjAttr{}, suite.attrCache.cacheMap[truncatedPath].attr) - suite.assert.True(suite.attrCache.cacheMap[truncatedPath].valid()) - suite.assert.False(suite.attrCache.cacheMap[truncatedPath].exists()) - suite.assert.NotNil(suite.attrCache.cacheMap[truncatedPath].cachedAt) + item := getCacheItem(suite.attrCache, truncatedPath) + suite.assert.NotNil(item) + suite.assert.Nil(item.attr) + suite.assert.False(item.exists) + suite.assert.False(item.cachedAt.IsZero()) }) } } +func (suite *attrCacheTestSuite) TestGetAttrBareErrNotExistCachesNegativeEntry() { + defer suite.cleanupTest() + path := "a" + options := internal.GetAttrOptions{Name: path} + suite.mock.EXPECT().GetAttr(options).Return(&internal.ObjAttr{}, os.ErrNotExist) + + _, err := suite.attrCache.GetAttr(options) + suite.assert.Equal(os.ErrNotExist, err) + item := getCacheItem(suite.attrCache, path) + suite.assert.NotNil(item) + suite.assert.True(item.isNegativeEntry()) +} + +func (suite *attrCacheTestSuite) TestGetAttrPathErrorCachesNegativeEntry() { + defer suite.cleanupTest() + path := "a" + pathErr := &fs.PathError{Op: "stat", Path: path, Err: syscall.ENOENT} + options := internal.GetAttrOptions{Name: path} + suite.mock.EXPECT().GetAttr(options).Return(&internal.ObjAttr{}, pathErr) + + _, err := suite.attrCache.GetAttr(options) + suite.assert.Equal(pathErr, err) + item := getCacheItem(suite.attrCache, path) + suite.assert.NotNil(item) + suite.assert.True(item.isNegativeEntry()) +} + +func (suite *attrCacheTestSuite) TestGetAttrSyscallEnoentCachesNegativeEntry() { + defer suite.cleanupTest() + path := "a" + options := internal.GetAttrOptions{Name: path} + suite.mock.EXPECT().GetAttr(options).Return(&internal.ObjAttr{}, syscall.ENOENT) + + _, err := suite.attrCache.GetAttr(options) + suite.assert.Equal(syscall.ENOENT, err) + item := getCacheItem(suite.attrCache, path) + suite.assert.NotNil(item) + suite.assert.True(item.isNegativeEntry()) +} + // Tests Cache Timeout func (suite *attrCacheTestSuite) TestCacheTimeout() { defer suite.cleanupTest() @@ -1043,14 +1086,14 @@ func (suite *attrCacheTestSuite) TestCacheTimeout() { cacheTimeout := 1 config := fmt.Sprintf("attr_cache:\n timeout-sec: %d", cacheTimeout) suite.setupTestHelper(config) // setup a new attr cache with a custom config (clean up will occur after the test as usual) - suite.assert.EqualValues(suite.attrCache.cacheTimeout, cacheTimeout) + suite.assert.Equal(time.Duration(cacheTimeout)*time.Second, suite.attrCache.cacheTimeout) path := "a" options := internal.GetAttrOptions{Name: path} // attributes should not be accessible so call the mock suite.mock.EXPECT().GetAttr(options).Return(getPathAttr(path, defaultSize, fs.FileMode(defaultMode), true), nil) - suite.assert.Empty(suite.attrCache.cacheMap) // cacheMap should be empty before call + suite.assert.Zero(suite.attrCache.lru.Len()) // cache should be empty before call _, err := suite.attrCache.GetAttr(options) suite.assert.NoError(err) assertUntouched(suite, path) // item added to cache after @@ -1068,107 +1111,6 @@ func (suite *attrCacheTestSuite) TestCacheTimeout() { suite.assert.NoError(err) } -// Tests Cache Cleanup - expired entries are actually removed from cache map -func (suite *attrCacheTestSuite) TestCacheCleanupExpiredEntries() { - defer suite.cleanupTest() - suite.cleanupTest() // clean up the default attr cache generated - cacheTimeout := 2 - config := fmt.Sprintf("attr_cache:\n timeout-sec: %d", cacheTimeout) - suite.setupTestHelper(config) // setup a new attr cache with a custom config - suite.assert.EqualValues(suite.attrCache.cacheTimeout, cacheTimeout) - - path1 := "file1" - path2 := "file2" - options1 := internal.GetAttrOptions{Name: path1} - options2 := internal.GetAttrOptions{Name: path2} - - // Add two files to cache - suite.mock.EXPECT().GetAttr(options1).Return(getPathAttr(path1, defaultSize, fs.FileMode(defaultMode), true), nil) - suite.mock.EXPECT().GetAttr(options2).Return(getPathAttr(path2, defaultSize, fs.FileMode(defaultMode), true), nil) - - suite.assert.Empty(suite.attrCache.cacheMap) // cacheMap should be empty before calls - _, err := suite.attrCache.GetAttr(options1) - suite.assert.NoError(err) - _, err = suite.attrCache.GetAttr(options2) - suite.assert.NoError(err) - - // Verify both items are in cache - suite.assert.Len(suite.attrCache.cacheMap, 2) - assertUntouched(suite, path1) - assertUntouched(suite, path2) - - // Wait for cache timeout to expire, plus additional time for background cleanup to run - time.Sleep(time.Second * time.Duration(cacheTimeout+1)) - - // Verify that the cache has been cleaned up by background cleanup - // Wait a bit more if cleanup is still in progress - maxWait := 3 * time.Second - waitInterval := 100 * time.Millisecond - waited := time.Duration(0) - - for waited < maxWait { - suite.attrCache.cacheLock.RLock() - cacheSize := len(suite.attrCache.cacheMap) - suite.attrCache.cacheLock.RUnlock() - - if cacheSize == 0 { - break - } - - time.Sleep(waitInterval) - waited += waitInterval - } - - // Verify that expired entries have been cleaned up - suite.assert.Empty(suite.attrCache.cacheMap) -} - -// Tests Cache Cleanup during bulk caching operations -func (suite *attrCacheTestSuite) TestCacheCleanupDuringBulkCaching() { - defer suite.cleanupTest() - suite.cleanupTest() // clean up the default attr cache generated - cacheTimeout := 3 // Use a longer timeout for this test - config := fmt.Sprintf("attr_cache:\n timeout-sec: %d", cacheTimeout) - suite.setupTestHelper(config) // setup a new attr cache with a custom config - suite.assert.EqualValues(suite.attrCache.cacheTimeout, cacheTimeout) - - // Add some items to cache manually with old timestamps - path1 := "oldfile1" - path2 := "oldfile2" - oldTime := time.Now().Add(-time.Second * time.Duration(cacheTimeout+1)) - suite.attrCache.cacheMap[path1] = newAttrCacheItem(getPathAttr(path1, defaultSize, fs.FileMode(defaultMode), true), true, oldTime) - suite.attrCache.cacheMap[path2] = newAttrCacheItem(getPathAttr(path2, defaultSize, fs.FileMode(defaultMode), true), true, oldTime) - - // Verify both old items are in cache - suite.assert.Len(suite.attrCache.cacheMap, 2) - - // Wait a bit for background cleanup to run and remove expired items - time.Sleep(time.Second * time.Duration(cacheTimeout+1)) - - // Wait for cleanup to complete - maxWait := 2 * time.Second - waitInterval := 100 * time.Millisecond - waited := time.Duration(0) - - for waited < maxWait { - suite.attrCache.cacheLock.RLock() - cacheSize := len(suite.attrCache.cacheMap) - suite.attrCache.cacheLock.RUnlock() - - if cacheSize == 0 { - break - } - - time.Sleep(waitInterval) - waited += waitInterval - } - - // Verify that expired entries have been cleaned up - suite.assert.Empty(suite.attrCache.cacheMap) - suite.assert.NotContains(suite.attrCache.cacheMap, path1) - suite.assert.NotContains(suite.attrCache.cacheMap, path2) -} - // Tests CreateLink func (suite *attrCacheTestSuite) TestCreateLink() { defer suite.cleanupTest() @@ -1182,8 +1124,8 @@ func (suite *attrCacheTestSuite) TestCreateLink() { err := suite.attrCache.CreateLink(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, link) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(link)) + suite.assert.False(suite.attrCache.lru.Has(path)) // Success // Entry Does Not Already Exist @@ -1191,8 +1133,8 @@ func (suite *attrCacheTestSuite) TestCreateLink() { err = suite.attrCache.CreateLink(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, link) - suite.assert.NotContains(suite.attrCache.cacheMap, path) + suite.assert.False(suite.attrCache.lru.Has(link)) + suite.assert.False(suite.attrCache.lru.Has(path)) // Entry Already Exists addPathToCache(suite.assert, suite.attrCache, link, false) @@ -1224,7 +1166,7 @@ func (suite *attrCacheTestSuite) TestChmod() { err := suite.attrCache.Chmod(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Success // Entry Does Not Already Exist @@ -1232,7 +1174,7 @@ func (suite *attrCacheTestSuite) TestChmod() { err = suite.attrCache.Chmod(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Entry Already Exists addPathToCache(suite.assert, suite.attrCache, path, false) @@ -1240,12 +1182,12 @@ func (suite *attrCacheTestSuite) TestChmod() { err = suite.attrCache.Chmod(options) suite.assert.NoError(err) - suite.assert.Contains(suite.attrCache.cacheMap, truncatedPath) - suite.assert.NotEqual(&internal.ObjAttr{}, suite.attrCache.cacheMap[truncatedPath].attr) - suite.assert.Equal(suite.attrCache.cacheMap[truncatedPath].attr.Size, defaultSize) - suite.assert.Equal(suite.attrCache.cacheMap[truncatedPath].attr.Mode, mode) // new mode should be set - suite.assert.True(suite.attrCache.cacheMap[truncatedPath].valid()) - suite.assert.True(suite.attrCache.cacheMap[truncatedPath].exists()) + item := getCacheItem(suite.attrCache, truncatedPath) + suite.assert.NotNil(item) + suite.assert.NotEqual(&internal.ObjAttr{}, item.attr) + suite.assert.Equal(item.attr.Size, defaultSize) + suite.assert.Equal(item.attr.Mode, mode) // new mode should be set + suite.assert.True(item.exists) }) } } @@ -1271,7 +1213,7 @@ func (suite *attrCacheTestSuite) TestChown() { err := suite.attrCache.Chown(options) suite.assert.Error(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Success // Entry Does Not Already Exist @@ -1279,7 +1221,7 @@ func (suite *attrCacheTestSuite) TestChown() { err = suite.attrCache.Chown(options) suite.assert.NoError(err) - suite.assert.NotContains(suite.attrCache.cacheMap, truncatedPath) + suite.assert.False(suite.attrCache.lru.Has(truncatedPath)) // Entry Already Exists addPathToCache(suite.assert, suite.attrCache, path, false) @@ -1292,8 +1234,223 @@ func (suite *attrCacheTestSuite) TestChown() { } } +// TestLRUEvictionOnMemoryLimit verifies that the LRU evicts the least-recently-used entry +// when the cache exceeds its configured memory limit. +func (suite *attrCacheTestSuite) TestLRUEvictionOnMemoryLimit() { + defer suite.cleanupTest() + suite.cleanupTest() + + // setupTestHelper creates and starts the cache. + suite.setupTestHelper(emptyConfig) + + // Insert one entry to measure its cost, then rebuild the LRU capped at 2 entries. + path0 := "measure" + item0 := &attrCacheItem{attr: getPathAttr(path0, defaultSize, fs.FileMode(defaultMode), false), exists: true, cachedAt: time.Now()} + suite.attrCache.lru.Put(path0, item0) + singleEntrySize := suite.attrCache.lru.Size() + // Set max to exactly 2 entries. + suite.attrCache.lru = newAttrCacheLRU(singleEntrySize * 2) + + // Add 3 entries (A, B, C in insertion order: A is LRU). + pathA, pathB, pathLast := "lru_a", "lru_b", "lru_c" + for _, p := range []string{pathA, pathB, pathLast} { + item := &attrCacheItem{attr: getPathAttr(p, defaultSize, fs.FileMode(defaultMode), false), exists: true, cachedAt: time.Now()} + suite.attrCache.lru.Put(p, item) + } + + // Only 2 entries should remain, and A (the LRU) should have been evicted. + suite.assert.Equal(2, suite.attrCache.lru.Len()) + suite.assert.False(suite.attrCache.lru.Has(pathA), "LRU entry should have been evicted") + suite.assert.True(suite.attrCache.lru.Has(pathB)) + suite.assert.True(suite.attrCache.lru.Has(pathLast)) + suite.assert.LessOrEqual(suite.attrCache.lru.Size(), singleEntrySize*2) +} + +// TestLRUOrderPreservesRecentlyAccessed verifies that a recently-accessed entry +// survives eviction over an older, less-recently-used entry. +func (suite *attrCacheTestSuite) TestLRUOrderPreservesRecentlyAccessed() { + defer suite.cleanupTest() + suite.cleanupTest() + + suite.setupTestHelper(emptyConfig) + + // Measure single entry size and set limit to exactly 2 entries. + path0 := "measure" + item0 := &attrCacheItem{attr: getPathAttr(path0, defaultSize, fs.FileMode(defaultMode), false), exists: true, cachedAt: time.Now()} + suite.attrCache.lru.Put(path0, item0) + singleEntrySize := suite.attrCache.lru.Size() + suite.attrCache.lru = newAttrCacheLRU(singleEntrySize * 2) + + // Insert A, then B. Order: B (MRU) → A (LRU). + pathA, pathB := "ord_a", "ord_b" + for _, p := range []string{pathA, pathB} { + item := &attrCacheItem{attr: getPathAttr(p, defaultSize, fs.FileMode(defaultMode), false), exists: true, cachedAt: time.Now()} + suite.attrCache.lru.Put(p, item) + } + + // Access A via GetAttr — this promotes A to MRU. Order: A (MRU) → B (LRU). + // pathA is valid in cache; GetAttr serves it without calling the mock. + _, err := suite.attrCache.GetAttr(internal.GetAttrOptions{Name: pathA}) + suite.assert.NoError(err) + + // Add C — this should evict B (now LRU), not A. + pathLast := "ord_c" + itemLast := &attrCacheItem{attr: getPathAttr(pathLast, defaultSize, fs.FileMode(defaultMode), false), exists: true, cachedAt: time.Now()} + suite.attrCache.lru.Put(pathLast, itemLast) + + suite.assert.Equal(2, suite.attrCache.lru.Len()) + suite.assert.True(suite.attrCache.lru.Has(pathA), "A was recently accessed and should survive") + suite.assert.False(suite.attrCache.lru.Has(pathB), "B should be evicted (LRU)") + suite.assert.True(suite.attrCache.lru.Has(pathLast)) +} + // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestAttrCacheTestSuite(t *testing.T) { suite.Run(t, new(attrCacheTestSuite)) } + +// ---- TTL sweeper tests ---- + +type sweeperTestSuite struct { + suite.Suite + assert *assert.Assertions +} + +func (s *sweeperTestSuite) SetupTest() { + s.assert = assert.New(s.T()) +} + +// newSweeperCache builds a minimal AttrCache with no mock next-component, +// suitable for testing sweepExpired directly. +func newSweeperCache(timeout time.Duration) *AttrCache { + ac := &AttrCache{cacheTimeout: timeout} + ac.lru = newAttrCacheLRU(1 * 1024 * 1024) + return ac +} + +func expiredItem() *attrCacheItem { + return &attrCacheItem{cachedAt: time.Now().Add(-1 * time.Hour), exists: true, attr: makeAttr("x")} +} + +func freshItem() *attrCacheItem { + return &attrCacheItem{cachedAt: time.Now(), exists: true, attr: makeAttr("x")} +} + +// TestSweepExpiredRemovesExpiredEntries verifies that entries past their TTL are deleted +// and fresh entries within TTL are kept. +func (s *sweeperTestSuite) TestSweepExpiredRemovesExpiredEntries() { + ac := newSweeperCache(100 * time.Millisecond) + ac.lru.Put("expired", &attrCacheItem{cachedAt: time.Now().Add(-200 * time.Millisecond), exists: true, attr: makeAttr("expired")}) + ac.lru.Put("fresh", freshItem()) + + ac.sweepExpired() + + s.assert.Equal(1, ac.lru.Len()) + _, ok := ac.lru.Peek("expired") + s.assert.False(ok) + _, ok = ac.lru.Peek("fresh") + s.assert.True(ok) +} + +// TestSweepExpiredKeepsFreshEntries verifies that no entries are removed when none have expired. +func (s *sweeperTestSuite) TestSweepExpiredKeepsFreshEntries() { + ac := newSweeperCache(1 * time.Hour) + ac.lru.Put("a", freshItem()) + ac.lru.Put("b", freshItem()) + + ac.sweepExpired() + + s.assert.Equal(2, ac.lru.Len()) +} + +// TestSweepExpiredRemovesNegativeEntries verifies that expired tombstones are swept too. +func (s *sweeperTestSuite) TestSweepExpiredRemovesNegativeEntries() { + ac := newSweeperCache(100 * time.Millisecond) + ac.lru.Put("tombstone", &attrCacheItem{cachedAt: time.Now().Add(-200 * time.Millisecond), exists: false}) + + ac.sweepExpired() + + s.assert.Equal(0, ac.lru.Len()) +} + +// TestSweepExpiredSkipsWhenCacheIsActive verifies the idle gate: if lastOp is recent +// (within cacheTimeout/2), the sweep is skipped entirely. +func (s *sweeperTestSuite) TestSweepExpiredSkipsWhenCacheIsActive() { + ac := newSweeperCache(1 * time.Hour) + ac.lru.Put("expired", expiredItem()) + + // Simulate recent cache activity — well within the cacheTimeout/2 idle gate. + ac.lru.lastOp.Store(time.Now().Unix()) + + ac.sweepExpired() + + s.assert.Equal(1, ac.lru.Len(), "sweep must be skipped when cache is active") +} + +// TestSweepExpiredRunsWhenCacheIsIdle verifies that entries are swept when lastOp +// indicates the cache has been idle for longer than cacheTimeout/2. +func (s *sweeperTestSuite) TestSweepExpiredRunsWhenCacheIsIdle() { + ac := newSweeperCache(100 * time.Millisecond) + ac.lru.Put("expired", &attrCacheItem{cachedAt: time.Now().Add(-200 * time.Millisecond)}) + + // Simulate the cache being idle for 2 hours. + ac.lru.lastOp.Store(time.Now().Add(-2 * time.Hour).Unix()) + + ac.sweepExpired() + + s.assert.Equal(0, ac.lru.Len()) +} + +// TestSweepExpiredNoopWhenTimeoutZero verifies that sweep is a no-op when cacheTimeout == 0, +// preventing all entries from being treated as instantly expired. +func (s *sweeperTestSuite) TestSweepExpiredNoopWhenTimeoutZero() { + ac := newSweeperCache(0) + ac.lru.Put("entry", freshItem()) + + ac.sweepExpired() + + s.assert.Equal(1, ac.lru.Len()) +} + +// TestSweeperGoroutineEvictsEntries is an integration test that starts the sweeper goroutine +// and verifies it removes expired entries within a few sweep cycles. +func (s *sweeperTestSuite) TestSweeperGoroutineEvictsEntries() { + const timeout = 50 * time.Millisecond + ac := &AttrCache{cacheTimeout: timeout} + _ = ac.Start(context.Background()) + defer func() { _ = ac.Stop() }() + + ac.lru.cachePositiveEntry("file", makeAttr("file")) + + // Wait for the entry to expire and at least two sweep cycles to complete. + time.Sleep(4 * timeout) + + s.assert.Equal(0, ac.lru.Len(), "sweeper goroutine must have removed the expired entry") +} + +// TestGetAttrUpdatesLastOp verifies that GetAttr refreshes the idle-gate timestamp, +// preventing the sweeper from running during active lookup traffic. +func (s *sweeperTestSuite) TestGetAttrUpdatesLastOp() { + ac := newSweeperCache(1 * time.Hour) + ac.SetNextComponent(&attrCacheNoopNext{}) + + before := time.Now() + _, _ = ac.GetAttr(internal.GetAttrOptions{Name: "any"}) + + stored := ac.lru.lastOp.Load() + s.assert.NotZero(stored) + s.assert.GreaterOrEqual(stored, before.Unix(), "lastOp must be updated by GetAttr") +} + +// attrCacheNoopNext is a minimal Component stub used by sweeper tests that need +// a non-nil next component but don't care about its behaviour. +type attrCacheNoopNext struct{ internal.BaseComponent } + +func (n *attrCacheNoopNext) GetAttr(_ internal.GetAttrOptions) (*internal.ObjAttr, error) { + return nil, os.ErrNotExist +} + +func TestSweeperTestSuite(t *testing.T) { + suite.Run(t, new(sweeperTestSuite)) +} diff --git a/component/attr_cache/cacheMap.go b/component/attr_cache/cacheMap.go deleted file mode 100644 index 24758f674..000000000 --- a/component/attr_cache/cacheMap.go +++ /dev/null @@ -1,111 +0,0 @@ -/* - _____ _____ _____ ____ ______ _____ ------ - | | | | | | | | | | | | | - | | | | | | | | | | | | | - | --- | | | | |-----| |---- | | |-----| |----- ------ - | | | | | | | | | | | | | - | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ - - - Licensed under the MIT License . - - Copyright © 2020-2026 Microsoft Corporation. All rights reserved. - Author : - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE -*/ - -package attr_cache - -import ( - "os" - "time" - - "github.com/Azure/azure-storage-fuse/v2/common" - "github.com/Azure/azure-storage-fuse/v2/internal" -) - -// Flags represented in BitMap for various flags in the attr cache item -const ( - AttrFlagUnknown uint64 = iota - AttrFlagExists - AttrFlagValid -) - -// attrCacheItem : Structure of each item in attr cache -type attrCacheItem struct { - attr *internal.ObjAttr - cachedAt time.Time - attrFlag common.BitMap64 -} - -func newAttrCacheItem(attr *internal.ObjAttr, exists bool, cachedAt time.Time) *attrCacheItem { - item := &attrCacheItem{ - attr: attr, - attrFlag: 0, - cachedAt: cachedAt, - } - - item.attrFlag.Set(AttrFlagValid) - if exists { - item.attrFlag.Set(AttrFlagExists) - } - - return item -} - -func (value *attrCacheItem) valid() bool { - return value.attrFlag.IsSet(AttrFlagValid) -} - -func (value *attrCacheItem) exists() bool { - return value.attrFlag.IsSet(AttrFlagExists) -} - -func (value *attrCacheItem) markDeleted(deletedTime time.Time) { - value.attrFlag.Clear(AttrFlagExists) - value.attrFlag.Set(AttrFlagValid) - value.cachedAt = deletedTime - value.attr = &internal.ObjAttr{} -} - -func (value *attrCacheItem) invalidate() { - value.attrFlag.Clear(AttrFlagValid) - value.attr = &internal.ObjAttr{} -} - -func (value *attrCacheItem) getAttr() *internal.ObjAttr { - return value.attr -} - -func (value *attrCacheItem) isDeleted() bool { - return !value.exists() -} - -func (value *attrCacheItem) setSize(size int64) { - value.attr.Mtime = time.Now() - value.attr.Size = size - value.cachedAt = time.Now() -} - -func (value *attrCacheItem) setMode(mode os.FileMode) { - value.attr.Mode = mode - value.attr.Ctime = time.Now() - value.cachedAt = time.Now() -} diff --git a/component/attr_cache/cache_item.go b/component/attr_cache/cache_item.go new file mode 100644 index 000000000..66abb4e78 --- /dev/null +++ b/component/attr_cache/cache_item.go @@ -0,0 +1,236 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2026 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package attr_cache + +import ( + "strings" + "sync/atomic" + "time" + "unsafe" + + cachepolicy "github.com/Azure/azure-storage-fuse/v2/common/cache_policy" + "github.com/Azure/azure-storage-fuse/v2/common/log" + "github.com/Azure/azure-storage-fuse/v2/internal" +) + +// attrCacheItem : Structure of each item in attr cache +type attrCacheItem struct { + attr *internal.ObjAttr + cachedAt time.Time + exists bool // true if the entry exists in storage; false marks a negative cache entry (path confirmed absent) +} + +func (value *attrCacheItem) isNegativeEntry() bool { + return !value.exists +} + +// positiveEntrySize is the approximate heap cost of a positive (non-tombstone) cache entry +// with a typical path (52 bytes), ETag (20 bytes), and MD5 (16 bytes), after GOGC doubling. +// Used to convert a legacy max-files count into an equivalent max-size-mb value. +const positiveEntrySize int64 = 964 + +// estimateAttrCacheEntrySize estimates the heap bytes for one attr-cache key-value pair. +// The result is doubled to account for Go's GOGC=100 behaviour where RSS ≈ 2× live heap. +// +// Typical sizes (52-byte path, 20-byte ETag, 16-byte MD5): +// - negative entry (tombstone): ~320 B/entry +// - positive entry (no metadata): ~964 B/entry +// +// See TestNegativeEntryCapacityMatchesEstimate and TestPositiveEntryCapacityMatchesEstimate +// in cache_item_test.go for exact accounting and how to recalculate for different path lengths. +func estimateAttrCacheEntrySize(key string, item *attrCacheItem) int64 { + // Key string data (the string header is already in LRU's lruItem struct) + sz := int64(len(key)) + + if item != nil { + // unsafe.Sizeof returns only the struct header size — it does not include + // heap-allocated data referenced by pointer fields (strings, slices, maps). + // Those are accounted for explicitly below. + sz += int64(unsafe.Sizeof(*item)) + + if item.attr != nil { + sz += int64(unsafe.Sizeof(*item.attr)) + + sz += int64(len(item.attr.Path)) + sz += int64(len(item.attr.Name)) + sz += int64(len(item.attr.ETag)) + sz += int64(len(item.attr.MD5)) + + for k, v := range item.attr.Metadata { + // 32 bytes covers the per-entry overhead of a Go map bucket: + // key pointer (8) + value pointer (8) + map internal bookkeeping (~16). + sz += 32 + int64(len(k)) + if v != nil { + sz += int64(len(*v)) + } + } + } + } + + // With GOGC=100 (Go's default), the runtime lets garbage accumulate up to 100% of the + // live heap before collecting, so RSS can reach ~2× the live heap. Doubling the estimate + // keeps the LRU limit meaningful in RSS terms rather than raw heap terms. + return sz * 2 +} + +// attrCacheLRU is an LRU cache specialised for attr-cache entries. Embedding the generic +// LRU promotes all its methods (Get, Put, Peek, Delete, …) while allowing cache-specific +// helpers to be defined here alongside the data model they operate on. +// lastOp records the Unix second of the last cache operation for the TTL sweeper's idle gate. +type attrCacheLRU struct { + *cachepolicy.LRU[string, *attrCacheItem] + lastOp atomic.Int64 +} + +func newAttrCacheLRU(maxSizeBytes int64) *attrCacheLRU { + return &attrCacheLRU{LRU: cachepolicy.NewLRU(maxSizeBytes, estimateAttrCacheEntrySize)} +} + +func (l *attrCacheLRU) touch() { + l.lastOp.Store(time.Now().Unix()) +} + +// Get promotes the entry to MRU and records cache activity for the idle gate. +func (l *attrCacheLRU) Get(key string) (*attrCacheItem, bool) { + l.touch() + return l.LRU.Get(key) +} + +// Has checks whether a key exists and records cache activity for the idle gate. +func (l *attrCacheLRU) Has(key string) bool { + l.touch() + return l.LRU.Has(key) +} + +// Put inserts or updates an entry and records cache activity for the idle gate. +// Returns false (without modifying the cache) if the entry's size exceeds maxCacheSize. +func (l *attrCacheLRU) Put(key string, val *attrCacheItem) bool { + l.touch() + return l.LRU.Put(key, val) +} + +// Delete removes an entry and records cache activity for the idle gate. +func (l *attrCacheLRU) Delete(key string) { + l.touch() + l.LRU.Delete(key) +} + +// ReplaceIf replaces matching entries and records cache activity for the idle gate. +// DeleteIf is intentionally not overridden because the TTL sweeper relies on it without +// resetting the idle gate. Call l.touch() explicitly before using DeleteIf elsewhere. +func (l *attrCacheLRU) ReplaceIf(pred func(string, *attrCacheItem) bool, newVal func(string) *attrCacheItem) { + l.touch() + l.LRU.ReplaceIf(pred, newVal) +} + +func (l *attrCacheLRU) cachePositiveEntry(path string, attr *internal.ObjAttr) { + if !l.Put(path, &attrCacheItem{attr: attr, exists: true, cachedAt: time.Now()}) { + log.Err("attrCacheLRU::cachePositiveEntry : entry too large for cache, skipping path %s", path) + } +} + +func (l *attrCacheLRU) cacheNegativeEntry(path string) { + if !l.Put(path, &attrCacheItem{cachedAt: time.Now()}) { + log.Err("attrCacheLRU::cacheNegativeEntry : entry too large for cache, skipping path %s", path) + } +} + +func (l *attrCacheLRU) cacheAttributes(pathList []*internal.ObjAttr) { + if len(pathList) == 0 { + return + } + // Bulk caching can involve thousands of entries; avoid a time.Now()/atomic.Store per item. + l.touch() + currTime := time.Now() + for _, attr := range pathList { + key := internal.TruncateDirName(attr.Path) + if !l.LRU.Put(key, &attrCacheItem{attr: attr, exists: true, cachedAt: currTime}) { + log.Err("attrCacheLRU::cacheAttributes : entry too large for cache, skipping path %s", key) + } + } +} + +// Marks the entry as negative +func (l *attrCacheLRU) deletePath(path string, t time.Time) { + if !l.Put(internal.TruncateDirName(path), &attrCacheItem{cachedAt: t}) { + log.Err("attrCacheLRU::deletePath : tombstone too large for cache, skipping path %s", path) + } +} + +// Removes the entry from the cache. +func (l *attrCacheLRU) invalidatePath(path string) { + l.Delete(internal.TruncateDirName(path)) +} + +func (l *attrCacheLRU) deleteDirectory(path string, t time.Time) { + // Mark all the child entries as negative entries + prefix := internal.ExtendDirName(path) + l.ReplaceIf(func(key string, _ *attrCacheItem) bool { + return strings.HasPrefix(key, prefix) + }, func(_ string) *attrCacheItem { + return &attrCacheItem{cachedAt: t} + }) + + // Mark the directory entry itself as a negative entry + l.deletePath(path, t) +} + +func (l *attrCacheLRU) invalidateDirectory(path string) { + // Invalidate all the child entries + l.touch() + prefix := internal.ExtendDirName(path) + l.DeleteIf(func(key string, _ *attrCacheItem) bool { + return strings.HasPrefix(key, prefix) + }) + + // Invalidate directory entry itself + l.invalidatePath(path) +} + +func (l *attrCacheLRU) refreshEntry(path string, attr *internal.ObjAttr) { + path = internal.TruncateDirName(path) + if attr == nil { + // Src attributes unavailable — invalidate Dst so the next GetAttr refetches + // from the next component rather than poisoning it with a negative entry. + l.Delete(path) + return + } + if l.Has(path) { + copied := *attr // copy so we don't mutate the caller's struct + copied.Path = path + if !l.Put(path, &attrCacheItem{attr: &copied, exists: true, cachedAt: time.Now()}) { + log.Err("attrCacheLRU::refreshEntry : entry too large for cache, skipping path %s", path) + } + } +} diff --git a/component/attr_cache/cache_item_test.go b/component/attr_cache/cache_item_test.go new file mode 100644 index 000000000..a8e8c3a86 --- /dev/null +++ b/component/attr_cache/cache_item_test.go @@ -0,0 +1,497 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2026 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package attr_cache + +import ( + "fmt" + "testing" + "time" + "unsafe" + + "github.com/Azure/azure-storage-fuse/v2/internal" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type cacheMapTestSuite struct { + suite.Suite + assert *assert.Assertions +} + +func (s *cacheMapTestSuite) SetupTest() { + s.assert = assert.New(s.T()) +} + +func makeAttr(path string) *internal.ObjAttr { + return &internal.ObjAttr{Path: path, Name: path, Size: 1024, Mode: 0755} +} + +// ---- attrCacheItem ---- + +func (s *cacheMapTestSuite) TestNewAttrCacheItemPositive() { + attr := makeAttr("f") + t := time.Now() + item := &attrCacheItem{attr: attr, exists: true, cachedAt: t} + + s.assert.True(item.exists) + s.assert.Equal(attr, item.attr) + s.assert.Equal(t, item.cachedAt) +} + +func (s *cacheMapTestSuite) TestNewAttrCacheItemNegative() { + t := time.Now() + item := &attrCacheItem{cachedAt: t} + + s.assert.False(item.exists) + s.assert.Nil(item.attr) + s.assert.Equal(t, item.cachedAt) +} + +func (s *cacheMapTestSuite) TestGetAttr() { + attr := makeAttr("f") + item := &attrCacheItem{attr: attr, exists: true, cachedAt: time.Now()} + s.assert.Equal(attr, item.attr) +} + +func (s *cacheMapTestSuite) TestIsNegativeEntry() { + positive := &attrCacheItem{attr: makeAttr("f"), exists: true, cachedAt: time.Now()} + s.assert.False(positive.isNegativeEntry()) + + negative := &attrCacheItem{cachedAt: time.Now()} + s.assert.True(negative.isNegativeEntry()) +} + +// ---- estimateAttrCacheEntrySize ---- + +func (s *cacheMapTestSuite) TestEstimateSizeNilItem() { + key := "somepath" + sz := estimateAttrCacheEntrySize(key, nil) + expected := int64(len(key)) + expected += expected + s.assert.Equal(expected, sz) +} + +func (s *cacheMapTestSuite) TestEstimateSizeNilAttr() { + key := "p" + item := &attrCacheItem{cachedAt: time.Now()} + sz := estimateAttrCacheEntrySize(key, item) + + base := int64(len(key)) + int64(unsafe.Sizeof(*item)) + expected := base * 2 + s.assert.Equal(expected, sz) +} + +func (s *cacheMapTestSuite) TestEstimateSizeWithAttr() { + key := "file" + attr := makeAttr("file") + item := &attrCacheItem{attr: attr, exists: true, cachedAt: time.Now()} + sz := estimateAttrCacheEntrySize(key, item) + + // Must be strictly larger than key + item struct alone + base := int64(len(key)) + int64(unsafe.Sizeof(*item)) + s.assert.Greater(sz, base) +} + +func (s *cacheMapTestSuite) TestEstimateSizeGrowsWithMetadata() { + key := "f" + attr := makeAttr("f") + item := &attrCacheItem{attr: attr, exists: true, cachedAt: time.Now()} + szWithout := estimateAttrCacheEntrySize(key, item) + + v := "value" + attr.Metadata = map[string]*string{"key": &v} + szWith := estimateAttrCacheEntrySize(key, item) + + s.assert.Greater(szWith, szWithout) +} + +func (s *cacheMapTestSuite) TestEstimateSizeGrowsWithLongerStrings() { + short := estimateAttrCacheEntrySize("a", &attrCacheItem{attr: makeAttr("a"), exists: true, cachedAt: time.Now()}) + long := estimateAttrCacheEntrySize("averylongpathname", &attrCacheItem{attr: makeAttr("averylongpathname"), exists: true, cachedAt: time.Now()}) + s.assert.Greater(long, short) +} + +// ---- attrCacheLRU ---- + +func newTestLRU() *attrCacheLRU { + return newAttrCacheLRU(0) // nil = no idle-gate tracking +} + +func (s *cacheMapTestSuite) TestCachePositiveEntry() { + lru := newTestLRU() + attr := makeAttr("file") + + lru.cachePositiveEntry("file", attr) + + item, ok := lru.Peek("file") + s.assert.True(ok) + s.assert.True(item.exists) + s.assert.Equal(attr, item.attr) +} + +func (s *cacheMapTestSuite) TestCacheNegativeEntry() { + lru := newTestLRU() + + lru.cacheNegativeEntry("missing") + + item, ok := lru.Peek("missing") + s.assert.True(ok) + s.assert.False(item.exists) + s.assert.Nil(item.attr) +} + +func (s *cacheMapTestSuite) TestCacheAttributes() { + lru := newTestLRU() + attrs := []*internal.ObjAttr{ + makeAttr("dir/a"), + makeAttr("dir/b"), + makeAttr("dir/c/"), + } + + lru.cacheAttributes(attrs) + + s.assert.Equal(3, lru.Len()) + _, ok := lru.Peek("dir/a") + s.assert.True(ok) + _, ok = lru.Peek("dir/b") + s.assert.True(ok) + // trailing slash is truncated + _, ok = lru.Peek("dir/c") + s.assert.True(ok) +} + +func (s *cacheMapTestSuite) TestCacheAttributesAllPositive() { + lru := newTestLRU() + lru.cacheAttributes([]*internal.ObjAttr{makeAttr("a"), makeAttr("b")}) + + for _, key := range []string{"a", "b"} { + item, ok := lru.Peek(key) + s.assert.True(ok, key) + s.assert.True(item.exists, key) + } +} + +func (s *cacheMapTestSuite) TestDeletePath() { + lru := newTestLRU() + lru.cachePositiveEntry("file", makeAttr("file")) + t := time.Now() + + lru.deletePath("file", t) + + item, ok := lru.Peek("file") + s.assert.True(ok) + s.assert.True(item.isNegativeEntry()) + s.assert.Nil(item.attr) + s.assert.Equal(t, item.cachedAt) +} + +func (s *cacheMapTestSuite) TestDeletePathAbsent() { + lru := newTestLRU() + t := time.Now() + + lru.deletePath("nonexistent", t) + + // Always inserts a tombstone — callers confirmed the path is gone from storage + item, ok := lru.Peek("nonexistent") + s.assert.True(ok) + s.assert.True(item.isNegativeEntry()) + s.assert.Equal(t, item.cachedAt) +} + +func (s *cacheMapTestSuite) TestDeletePathTruncatesTrailingSlash() { + lru := newTestLRU() + lru.cachePositiveEntry("dir", makeAttr("dir")) + + lru.deletePath("dir/", time.Now()) + + item, ok := lru.Peek("dir") + s.assert.True(ok) + s.assert.True(item.isNegativeEntry()) +} + +func (s *cacheMapTestSuite) TestInvalidatePath() { + lru := newTestLRU() + lru.cachePositiveEntry("file", makeAttr("file")) + + lru.invalidatePath("file") + + _, ok := lru.Peek("file") + s.assert.False(ok) +} + +func (s *cacheMapTestSuite) TestInvalidatePathAbsent() { + lru := newTestLRU() + // Should not panic + lru.invalidatePath("nonexistent") + s.assert.Equal(0, lru.Len()) +} + +func (s *cacheMapTestSuite) TestInvalidatePathTruncatesTrailingSlash() { + lru := newTestLRU() + lru.cachePositiveEntry("dir", makeAttr("dir")) + + lru.invalidatePath("dir/") + + _, ok := lru.Peek("dir") + s.assert.False(ok) +} + +func (s *cacheMapTestSuite) TestDeleteDirectory() { + lru := newTestLRU() + lru.cachePositiveEntry("dir", makeAttr("dir")) + lru.cachePositiveEntry("dir/a", makeAttr("dir/a")) + lru.cachePositiveEntry("dir/b", makeAttr("dir/b")) + lru.cachePositiveEntry("other", makeAttr("other")) + + lru.deleteDirectory("dir", time.Now()) + + for _, key := range []string{"dir", "dir/a", "dir/b"} { + item, ok := lru.Peek(key) + s.assert.True(ok, key) + s.assert.True(item.isNegativeEntry(), key) + } + // sibling not affected + item, ok := lru.Peek("other") + s.assert.True(ok) + s.assert.False(item.isNegativeEntry()) +} + +func (s *cacheMapTestSuite) TestDeleteDirectoryPreservesNonChildren() { + lru := newTestLRU() + lru.cachePositiveEntry("dira", makeAttr("dira")) // "dira" != "dir/" + lru.cachePositiveEntry("dir/x", makeAttr("dir/x")) + + lru.deleteDirectory("dir", time.Now()) + + // "dira" starts with "dir" but not "dir/" — must be untouched + item, ok := lru.Peek("dira") + s.assert.True(ok) + s.assert.False(item.isNegativeEntry()) + + item, ok = lru.Peek("dir/x") + s.assert.True(ok) + s.assert.True(item.isNegativeEntry()) +} + +func (s *cacheMapTestSuite) TestDeleteDirectoryAlsoDeletesDirectoryItself() { + lru := newTestLRU() + lru.cachePositiveEntry("dir", makeAttr("dir")) + + lru.deleteDirectory("dir", time.Now()) + + item, ok := lru.Peek("dir") + s.assert.True(ok) + s.assert.True(item.isNegativeEntry()) +} + +func (s *cacheMapTestSuite) TestInvalidateDirectoryAlsoDeletesDirectoryItself() { + lru := newTestLRU() + lru.cachePositiveEntry("dir", makeAttr("dir")) + + lru.invalidateDirectory("dir") + + _, ok := lru.Peek("dir") + s.assert.False(ok) +} + +func (s *cacheMapTestSuite) TestInvalidateDirectory() { + lru := newTestLRU() + lru.cachePositiveEntry("dir", makeAttr("dir")) + lru.cachePositiveEntry("dir/a", makeAttr("dir/a")) + lru.cachePositiveEntry("dir/b", makeAttr("dir/b")) + lru.cachePositiveEntry("other", makeAttr("other")) + + lru.invalidateDirectory("dir") + + for _, key := range []string{"dir", "dir/a", "dir/b"} { + _, ok := lru.Peek(key) + s.assert.False(ok, key) + } + item, ok := lru.Peek("other") + s.assert.True(ok) + s.assert.True(item.exists) +} + +func (s *cacheMapTestSuite) TestInvalidateDirectoryPreservesNonChildren() { + lru := newTestLRU() + lru.cachePositiveEntry("dira", makeAttr("dira")) + lru.cachePositiveEntry("dir/x", makeAttr("dir/x")) + + lru.invalidateDirectory("dir") + + item, ok := lru.Peek("dira") + s.assert.True(ok) + s.assert.True(item.exists) + + _, ok = lru.Peek("dir/x") + s.assert.False(ok) +} + +func (s *cacheMapTestSuite) TestUpdateCacheEntry() { + lru := newTestLRU() + old := makeAttr("file") + lru.cachePositiveEntry("file", old) + + newAttr := &internal.ObjAttr{Size: 9999, Mode: 0600} + before := time.Now() + lru.refreshEntry("file", newAttr) + + item, ok := lru.Peek("file") + s.assert.True(ok) + s.assert.Equal(int64(9999), item.attr.Size) + s.assert.Equal("file", item.attr.Path) // path is set from the key + s.assert.True(item.exists) + s.assert.False(item.cachedAt.Before(before)) +} + +func (s *cacheMapTestSuite) TestUpdateCacheEntryAbsent() { + lru := newTestLRU() + // Should be a no-op for a path not in cache + lru.refreshEntry("missing", makeAttr("missing")) + s.assert.Equal(0, lru.Len()) +} + +// ---- Memory limit and capacity tests ---- + +// typicalPath returns a deterministic 52-byte path for index i, representative of +// real Azure blob paths (container + directory depth + filename). +// Fixed length keeps every entry's estimated size identical, making capacity +// arithmetic exact and regression-friendly. +// +// "logs/service/2024/" (18 B) + 34-digit zero-padded index (34 B) = 52 B +func typicalPath(i int) string { return fmt.Sprintf("logs/service/2024/%034d", i) } + +// typicalName returns the fixed-length filename portion of typicalPath. +func typicalName(i int) string { return fmt.Sprintf("%034d", i) } + +// fixedETag is a representative Azure ETag (quoted hex string, 20 bytes). +const fixedETag = "\"0x8DA1B2C3D4E5F6A7\"" + +// fixedMD5 is a representative 16-byte MD5 digest. +var fixedMD5 = []byte{ + 0x9b, 0x74, 0xc9, 0x89, 0x7b, 0xac, 0x77, 0x0f, + 0xfc, 0x9f, 0x8a, 0xd0, 0x7f, 0x36, 0x2e, 0x15, +} + +func makeTypicalAttr(i int) *internal.ObjAttr { + return &internal.ObjAttr{ + Path: typicalPath(i), + Name: typicalName(i), + ETag: fixedETag, + MD5: fixedMD5, + } +} + +// TestAttrCacheLRUHonorsMemoryLimit verifies the hard invariant: Size() never +// exceeds MaxSize() at any point during a fill that goes well past the limit. +func (s *cacheMapTestSuite) TestAttrCacheLRUHonorsMemoryLimit() { + const maxSize = 64 * 1024 * 1024 + lru := newAttrCacheLRU(maxSize) + + for i := 0; i < 200_000; i++ { + lru.cacheNegativeEntry(typicalPath(i)) + if i%10_000 == 0 { + s.assert.LessOrEqualf(lru.Size(), int64(maxSize), + "Size() exceeded MaxSize() after %d inserts", i+1) + } + } + s.assert.LessOrEqual(lru.Size(), int64(maxSize)) +} + +// TestNegativeEntryCapacityMatchesEstimate verifies that the actual number of +// entries the LRU holds at steady state equals the theoretical capacity derived +// from estimateAttrCacheEntrySize. A mismatch means the size estimator and the +// LRU accounting have drifted apart. +// Run with -v to see the per-entry byte budget and capacity for documentation. +func (s *cacheMapTestSuite) TestNegativeEntryCapacityMatchesEstimate() { + const maxSize = 64 * 1024 * 1024 + lru := newAttrCacheLRU(maxSize) + + k := typicalPath(0) + userSize := estimateAttrCacheEntrySize(k, &attrCacheItem{cachedAt: time.Now()}) + bytesPerEntry := lru.PerEntryOverhead() + userSize + expectedLen := int(int64(maxSize) / bytesPerEntry) + + s.T().Logf("negative entry: %d B/entry → 64 MB holds ~%d entries", bytesPerEntry, expectedLen) + + for i := 0; i < expectedLen*2; i++ { + lru.cacheNegativeEntry(typicalPath(i)) + } + + s.assert.Equal(expectedLen, lru.Len(), + "negative-entry capacity mismatch: bytesPerEntry=%d expectedLen=%d", bytesPerEntry, expectedLen) +} + +// TestPositiveEntryCapacityMatchesEstimate is the same check for positive entries +// with realistic ETag and MD5 fields populated. +// Run with -v to see the per-entry byte budget and capacity for documentation. +func (s *cacheMapTestSuite) TestPositiveEntryCapacityMatchesEstimate() { + const maxSize = 64 * 1024 * 1024 + lru := newAttrCacheLRU(maxSize) + + probe := makeTypicalAttr(0) + userSize := estimateAttrCacheEntrySize(typicalPath(0), &attrCacheItem{attr: probe, exists: true, cachedAt: time.Now()}) + bytesPerEntry := lru.PerEntryOverhead() + userSize + expectedLen := int(int64(maxSize) / bytesPerEntry) + + s.T().Logf("positive entry (ETag+MD5, no metadata): %d B/entry → 64 MB holds ~%d entries", bytesPerEntry, expectedLen) + + for i := 0; i < expectedLen*2; i++ { + lru.cachePositiveEntry(typicalPath(i), makeTypicalAttr(i)) + } + + s.assert.Equal(expectedLen, lru.Len(), + "positive-entry capacity mismatch: bytesPerEntry=%d expectedLen=%d", bytesPerEntry, expectedLen) +} + +// TestEntrySizesAreInExpectedRange guards against silent struct bloat or formula +// changes. The bounds are intentionally loose (~2× headroom) so they survive +// minor struct additions while still catching large regressions. +func (s *cacheMapTestSuite) TestEntrySizesAreInExpectedRange() { + lru := newAttrCacheLRU(0) + + negUserSize := estimateAttrCacheEntrySize(typicalPath(0), &attrCacheItem{cachedAt: time.Now()}) + negTotal := lru.PerEntryOverhead() + negUserSize + s.assert.GreaterOrEqualf(negTotal, int64(300), "negative entry shrank below 300 B (got %d B)", negTotal) + s.assert.LessOrEqualf(negTotal, int64(600), "negative entry grew past 600 B (got %d B)", negTotal) + + posUserSize := estimateAttrCacheEntrySize(typicalPath(0), &attrCacheItem{attr: makeTypicalAttr(0), exists: true, cachedAt: time.Now()}) + posTotal := lru.PerEntryOverhead() + posUserSize + s.assert.GreaterOrEqualf(posTotal, int64(700), "positive entry shrank below 700 B (got %d B)", posTotal) + s.assert.LessOrEqualf(posTotal, int64(1500), "positive entry grew past 1500 B (got %d B)", posTotal) +} + +func TestCacheMapTestSuite(t *testing.T) { + suite.Run(t, new(cacheMapTestSuite)) +} diff --git a/sampleBlockCacheConfig.yaml b/sampleBlockCacheConfig.yaml index efddda03c..4ad496bc6 100644 --- a/sampleBlockCacheConfig.yaml +++ b/sampleBlockCacheConfig.yaml @@ -23,6 +23,10 @@ block_cache: attr_cache: timeout-sec: 7200 + # max-size-mb controls how much memory the attribute cache may use. + # Default: auto-tuned to 1% of total RAM, clamped to [64 MB, 1 GB]. + # Set to 0 or omit to use the auto-tuned value. + # max-size-mb: 256 azstorage: type: block diff --git a/sampleFileCacheConfig.yaml b/sampleFileCacheConfig.yaml index d9a32766b..f4491fc21 100644 --- a/sampleFileCacheConfig.yaml +++ b/sampleFileCacheConfig.yaml @@ -22,6 +22,10 @@ file_cache: attr_cache: timeout-sec: 7200 + # max-size-mb controls how much memory the attribute cache may use. + # Default: auto-tuned to 1% of total RAM, clamped to [64 MB, 1 GB]. + # Set to 0 or omit to use the auto-tuned value. + # max-size-mb: 256 azstorage: type: block diff --git a/sampleFileCacheWithSASConfig.yaml b/sampleFileCacheWithSASConfig.yaml index b40f3ea17..0b506a2b8 100644 --- a/sampleFileCacheWithSASConfig.yaml +++ b/sampleFileCacheWithSASConfig.yaml @@ -22,6 +22,10 @@ file_cache: attr_cache: timeout-sec: 7200 + # max-size-mb controls how much memory the attribute cache may use. + # Default: auto-tuned to 1% of total RAM, clamped to [64 MB, 1 GB]. + # Set to 0 or omit to use the auto-tuned value. + # max-size-mb: 256 azstorage: type: block diff --git a/setup/advancedConfig.yaml b/setup/advancedConfig.yaml index efe1342b5..25b048561 100644 --- a/setup/advancedConfig.yaml +++ b/setup/advancedConfig.yaml @@ -113,6 +113,7 @@ file_cache: attr_cache: timeout-sec: