From 31a46540c2d7202d432d991a16d067992ea53b83 Mon Sep 17 00:00:00 2001 From: schletti2000 <29952256+schletti2000@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:58:59 +0200 Subject: [PATCH] Add an S3-compatible remote cache storage plugin Microsoft.MSBuildCache.S3 stores content-addressed outputs and fingerprint metadata in Amazon S3 or an S3-compatible endpoint, with a local BuildXL cache for repeated materialization. --- Directory.Packages.props | 1 + MSBuildCache.sln | 14 + README.md | 42 ++ src/Common/HexUtilities.cs | 22 + .../Microsoft.MSBuildCache.S3.Tests.csproj | 14 + src/S3.Tests/S3ObjectKeysTests.cs | 154 ++++ src/S3/MSBuildCacheS3Plugin.cs | 150 ++++ src/S3/Microsoft.MSBuildCache.S3.csproj | 41 + src/S3/S3CacheClient.cs | 706 ++++++++++++++++++ src/S3/S3ObjectKeys.cs | 90 +++ src/S3/S3PluginSettings.cs | 55 ++ src/S3/build/Microsoft.MSBuildCache.S3.props | 12 + .../build/Microsoft.MSBuildCache.S3.targets | 27 + .../Microsoft.MSBuildCache.S3.props | 3 + .../Microsoft.MSBuildCache.S3.targets | 3 + tests/TestProject/TestProject.csproj | 35 + tests/lib.ps1 | 18 +- tests/s3.ps1 | 338 +++++++++ 18 files changed, 1723 insertions(+), 2 deletions(-) create mode 100644 src/S3.Tests/Microsoft.MSBuildCache.S3.Tests.csproj create mode 100644 src/S3.Tests/S3ObjectKeysTests.cs create mode 100644 src/S3/MSBuildCacheS3Plugin.cs create mode 100644 src/S3/Microsoft.MSBuildCache.S3.csproj create mode 100644 src/S3/S3CacheClient.cs create mode 100644 src/S3/S3ObjectKeys.cs create mode 100644 src/S3/S3PluginSettings.cs create mode 100644 src/S3/build/Microsoft.MSBuildCache.S3.props create mode 100644 src/S3/build/Microsoft.MSBuildCache.S3.targets create mode 100644 src/S3/buildMultitargeting/Microsoft.MSBuildCache.S3.props create mode 100644 src/S3/buildMultitargeting/Microsoft.MSBuildCache.S3.targets create mode 100644 tests/s3.ps1 diff --git a/Directory.Packages.props b/Directory.Packages.props index a17e2fe..17c1ffb 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,6 +8,7 @@ true + diff --git a/MSBuildCache.sln b/MSBuildCache.sln index 85d9f83..6a4615a 100644 --- a/MSBuildCache.sln +++ b/MSBuildCache.sln @@ -32,10 +32,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Azur EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Local", "src\Local\Microsoft.MSBuildCache.Local.csproj", "{F6586428-E047-42C8-B0AC-048DF6DFAF18}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.S3", "src\S3\Microsoft.MSBuildCache.S3.csproj", "{9AF00174-D361-4143-8233-5A1A52965261}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Repack.Tests", "src\Repack.Tests\Microsoft.MSBuildCache.Repack.Tests.csproj", "{3BCB6452-B087-4A03-8418-C79F2715DDE7}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.AzurePipelines.Tests", "src\AzurePipelines.Tests\Microsoft.MSBuildCache.AzurePipelines.Tests.csproj", "{61A86AEA-F043-4CC4-B60B-A040C5C36194}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.S3.Tests", "src\S3.Tests\Microsoft.MSBuildCache.S3.Tests.csproj", "{1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -66,6 +70,10 @@ Global {F6586428-E047-42C8-B0AC-048DF6DFAF18}.Debug|x64.Build.0 = Debug|x64 {F6586428-E047-42C8-B0AC-048DF6DFAF18}.Release|x64.ActiveCfg = Release|x64 {F6586428-E047-42C8-B0AC-048DF6DFAF18}.Release|x64.Build.0 = Release|x64 + {9AF00174-D361-4143-8233-5A1A52965261}.Debug|x64.ActiveCfg = Debug|x64 + {9AF00174-D361-4143-8233-5A1A52965261}.Debug|x64.Build.0 = Debug|x64 + {9AF00174-D361-4143-8233-5A1A52965261}.Release|x64.ActiveCfg = Release|x64 + {9AF00174-D361-4143-8233-5A1A52965261}.Release|x64.Build.0 = Release|x64 {3BCB6452-B087-4A03-8418-C79F2715DDE7}.Debug|x64.ActiveCfg = Debug|x64 {3BCB6452-B087-4A03-8418-C79F2715DDE7}.Debug|x64.Build.0 = Debug|x64 {3BCB6452-B087-4A03-8418-C79F2715DDE7}.Release|x64.ActiveCfg = Release|x64 @@ -74,6 +82,10 @@ Global {61A86AEA-F043-4CC4-B60B-A040C5C36194}.Debug|x64.Build.0 = Debug|x64 {61A86AEA-F043-4CC4-B60B-A040C5C36194}.Release|x64.ActiveCfg = Release|x64 {61A86AEA-F043-4CC4-B60B-A040C5C36194}.Release|x64.Build.0 = Release|x64 + {1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}.Debug|x64.ActiveCfg = Debug|x64 + {1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}.Debug|x64.Build.0 = Debug|x64 + {1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}.Release|x64.ActiveCfg = Release|x64 + {1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -85,8 +97,10 @@ Global {D0195D37-E001-4283-B51A-A0B51B1D54D1} = {EFFB5949-347C-4F28-8964-571D5C6B6209} {97357681-C75E-445D-8547-46F312D01CED} = {EFFB5949-347C-4F28-8964-571D5C6B6209} {F6586428-E047-42C8-B0AC-048DF6DFAF18} = {EFFB5949-347C-4F28-8964-571D5C6B6209} + {9AF00174-D361-4143-8233-5A1A52965261} = {EFFB5949-347C-4F28-8964-571D5C6B6209} {3BCB6452-B087-4A03-8418-C79F2715DDE7} = {EFFB5949-347C-4F28-8964-571D5C6B6209} {61A86AEA-F043-4CC4-B60B-A040C5C36194} = {EFFB5949-347C-4F28-8964-571D5C6B6209} + {1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10} = {EFFB5949-347C-4F28-8964-571D5C6B6209} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {F1CDA78F-A666-431B-BF44-56DA7DF193BA} diff --git a/README.md b/README.md index c2e3778..6b1b882 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,48 @@ In the cases where an Azure credential is acquired, the following methods will b 3. If `$(MSBuildCacheManagedIdentityClientId)` is set, it will be used as a user-assigned [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview). This is recommended for non-interactive scenarios. 4. If `$(MSBuildCacheAllowInteractiveAuth)` is true, credentials will be obtained interactively. This is recommended for developer scenarios. +### Microsoft.MSBuildCache.S3 +[![NuGet Version](https://img.shields.io/nuget/v/Microsoft.MSBuildCache.S3.svg)](https://www.nuget.org/packages/Microsoft.MSBuildCache.S3) +[![NuGet Downloads](https://img.shields.io/nuget/dt/Microsoft.MSBuildCache.S3.svg)](https://www.nuget.org/packages/Microsoft.MSBuildCache.S3) + +This implementation uses [Amazon S3](https://aws.amazon.com/s3/), or an S3-compatible store such as [MinIO](https://min.io/), as the cache storage. + +> [!WARNING] +> This implementation does not yet have a robust security model. All builds using this will need write access to the bucket, so for example an external contributor could send a PR which would write/overwrite arbitrary content which could then be used by CI builds. Builds using this plugin must be restricted to trusted team members. Use at your own risk. + +These settings are available in addition to the [Common Settings](#common-settings): + +| MSBuild Property Name | Setting Type | Default value | Description | +| ------------- | ------------ | ------------- | ----------- | +| `$(MSBuildCacheS3BucketName)` | `string` | | The bucket used for cache storage. Required. | +| `$(MSBuildCacheS3Region)` | `string` | "us-east-1" | The AWS region. Also used as the signing region when `$(MSBuildCacheS3ServiceUrl)` is set. | +| `$(MSBuildCacheS3ServiceUrl)` | `Uri` | | The service url of an S3-compatible store, for example `http://localhost:9000` for MinIO. When unset, the AWS endpoint for the region is used. | +| `$(MSBuildCacheS3ForcePathStyle)` | `bool` | true when `$(MSBuildCacheS3ServiceUrl)` is set, otherwise false | Whether to use path-style addressing. Most S3-compatible stores require this. | +| `$(MSBuildCacheS3KeyPrefix)` | `string` | "msbuildcache" | The key prefix under which all cache objects are stored. | +| `$(MSBuildCacheS3MultipartThresholdBytes)` | `long` | 33554432 (32 MB) | Objects at or above this size are transferred as multiple parts in parallel instead of as a single request. | +| `$(MSBuildCacheS3MultipartPartSizeBytes)` | `long` | 8388608 (8 MB) | The part size for multipart transfers. Values below S3's 5 MB minimum are raised to it. | +| `$(MSBuildCacheS3MaxConcurrentPartsPerObject)` | `int` | 8 | How many parts of a single object are transferred concurrently. Ranged requests across all multipart transfers are additionally bounded by `$(MSBuildCacheMaxConcurrentCacheContentOperations)`. | + +The bucket is expected to already exist. Credentials are deliberately not settings, since settings are configured through item metadata and so end up in MSBuild logs. They are resolved as follows, in priority order: +1. If [`AWSCredentials`](https://docs.aws.amazon.com/sdkfornet/v3/apidocs/items/Runtime/TAWSCredentials.html) are provided directly in the plugin's constructor, they are used. This only applies when using the programmatic project cache API. +2. Otherwise the [default AWS credential chain](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/creds-assign.html) is used, which covers the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables, the shared credentials file, and instance and task roles. + +For example, to use a local MinIO instance: + +```xml + + Microsoft.MSBuildCache.S3 + msbuildcache + http://localhost:9000 + +``` + +The end-to-end test for this plugin starts a local [moto](https://github.com/getmoto/moto) server, so it needs no AWS account and no container runtime, only [uv](https://docs.astral.sh/uv) or an installed `moto_server`: + +``` +.\tests\s3.ps1 +``` + ## Other Packages ### Microsoft.MSBuildCache.SharedCompilation diff --git a/src/Common/HexUtilities.cs b/src/Common/HexUtilities.cs index 7959592..5d778f8 100644 --- a/src/Common/HexUtilities.cs +++ b/src/Common/HexUtilities.cs @@ -2,6 +2,10 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +#if NETFRAMEWORK +using System.Globalization; +using System.Text; +#endif namespace Microsoft.MSBuildCache; @@ -23,6 +27,24 @@ public static byte[] HexToBytes(string? hex) ? Array.Empty() : HexToBytes(hex.AsSpan()); + /// + /// Formats bytes as an uppercase hexadecimal string without a '0x' prefix, ie the inverse of . + /// + public static string BytesToHex(ReadOnlySpan bytes) + { +#if NETFRAMEWORK + StringBuilder result = new(bytes.Length * 2); + foreach (byte b in bytes) + { + result.Append(b.ToString("X2", CultureInfo.InvariantCulture)); + } + + return result.ToString(); +#else + return Convert.ToHexString(bytes); +#endif + } + /// /// Parses hexadecimal strings the form '1234abcd' or '0x9876fedb' into /// an array of bytes. diff --git a/src/S3.Tests/Microsoft.MSBuildCache.S3.Tests.csproj b/src/S3.Tests/Microsoft.MSBuildCache.S3.Tests.csproj new file mode 100644 index 0000000..f20cb79 --- /dev/null +++ b/src/S3.Tests/Microsoft.MSBuildCache.S3.Tests.csproj @@ -0,0 +1,14 @@ + + + + x64 + $(Platform) + net9.0 + Microsoft.MSBuildCache.S3.Tests + + $(NoWarn);CA1515 + + + + + diff --git a/src/S3.Tests/S3ObjectKeysTests.cs b/src/S3.Tests/S3ObjectKeysTests.cs new file mode 100644 index 0000000..a84e57f --- /dev/null +++ b/src/S3.Tests/S3ObjectKeysTests.cs @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using BuildXL.Cache.ContentStore.Hashing; +using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.MSBuildCache.S3.Tests; + +[TestClass] +public class S3ObjectKeysTests +{ + private const string KeyPrefix = "msbuildcache"; + private const string Universe = "abc123"; + + private static readonly Fingerprint WeakFingerprint = new(new byte[] { 1, 2, 3, 4 }); + private static readonly ContentHash PathSetHash = new(HashType.Vso0, new byte[33]); + + private static S3ObjectKeys CreateKeys(string keyPrefix = KeyPrefix, HashType hashType = HashType.Vso0, string universe = Universe) + => new(keyPrefix, hashType, universe); + + private static StrongFingerprint CreateStrongFingerprint(byte[] strongFingerprintBytes) + => new(WeakFingerprint, new Selector(PathSetHash, strongFingerprintBytes)); + + [TestMethod] + public void KeysAreScopedByPrefixAndUniverse() + { + S3ObjectKeys keys = CreateKeys(); + + string casKey = keys.GetCas(PathSetHash); + + StringAssert.StartsWith(casKey, $"{KeyPrefix}/", StringComparison.Ordinal); + StringAssert.Contains(casKey, $"/{Universe}/", StringComparison.Ordinal); + StringAssert.Contains(casKey, "/cas/", StringComparison.Ordinal); + } + + [TestMethod] + public void KeysAreScopedByHashType() + { + string vso0 = CreateKeys(hashType: HashType.Vso0).GetCas(PathSetHash); + string sha256 = CreateKeys(hashType: HashType.SHA256).GetCas(PathSetHash); + + Assert.AreNotEqual(vso0, sha256); + } + + [TestMethod] + public void KeysAreScopedByUniverse() + { + string one = CreateKeys(universe: "one").GetCas(PathSetHash); + string two = CreateKeys(universe: "two").GetCas(PathSetHash); + + Assert.AreNotEqual(one, two); + } + + [TestMethod] + [DataRow("")] + [DataRow(" ")] + [DataRow("/")] + public void EmptyKeyPrefixFallsBackToDefault(string keyPrefix) + { + string casKey = CreateKeys(keyPrefix: keyPrefix).GetCas(PathSetHash); + + StringAssert.StartsWith(casKey, "msbuildcache/", StringComparison.Ordinal); + } + + [TestMethod] + [DataRow("prefix/")] + [DataRow("/prefix")] + [DataRow(" prefix ")] + public void KeyPrefixIsNormalized(string keyPrefix) + { + string casKey = CreateKeys(keyPrefix: keyPrefix).GetCas(PathSetHash); + + StringAssert.StartsWith(casKey, "prefix/", StringComparison.Ordinal); + Assert.IsFalse(casKey.Contains("//", StringComparison.Ordinal), $"Key '{casKey}' contains an empty segment."); + } + + [TestMethod] + public void SerializedHashesDoNotAppearInKeys() + { + // ':' is legal in an S3 key but breaks tooling which maps keys onto file paths. + string casKey = CreateKeys().GetCas(PathSetHash); + + Assert.IsFalse(casKey.Contains(':', StringComparison.Ordinal), $"Key '{casKey}' contains a colon."); + } + + [TestMethod] + public void EntryAndSelectorKeysDiffer() + { + S3ObjectKeys keys = CreateKeys(); + StrongFingerprint fingerprint = CreateStrongFingerprint(new byte[] { 42 }); + + Assert.AreNotEqual(keys.GetEntry(fingerprint), keys.GetSelector(fingerprint)); + } + + [TestMethod] + public void SelectorPrefixMatchesItsSelectors() + { + S3ObjectKeys keys = CreateKeys(); + StrongFingerprint fingerprint = CreateStrongFingerprint(new byte[] { 42 }); + + StringAssert.StartsWith(keys.GetSelector(fingerprint), keys.GetSelectorPrefix(WeakFingerprint), StringComparison.Ordinal); + } + + [TestMethod] + // A single zero byte is what the base cache client uses for its empty selector. + [DataRow(new byte[] { 0 })] + [DataRow(new byte[] { 0xDE, 0xAD, 0xBE, 0xEF })] + [DataRow(new byte[0])] + public void SelectorsRoundTrip(byte[] strongFingerprintBytes) + { + S3ObjectKeys keys = CreateKeys(); + StrongFingerprint fingerprint = CreateStrongFingerprint(strongFingerprintBytes); + + bool parsed = keys.TryParseSelector(keys.GetSelector(fingerprint), WeakFingerprint, out Selector selector); + + Assert.IsTrue(parsed); + Assert.AreEqual(fingerprint.Selector, selector); + } + + [TestMethod] + public void SelectorsOfAnotherWeakFingerprintAreRejected() + { + S3ObjectKeys keys = CreateKeys(); + string selectorKey = keys.GetSelector(CreateStrongFingerprint(new byte[] { 42 })); + + Assert.IsFalse(keys.TryParseSelector(selectorKey, new Fingerprint(new byte[] { 9, 9, 9, 9 }), out _)); + } + + [TestMethod] + public void SelectorsOfAnotherUniverseAreRejected() + { + string selectorKey = CreateKeys(universe: "other").GetSelector(CreateStrongFingerprint(new byte[] { 42 })); + + Assert.IsFalse(CreateKeys().TryParseSelector(selectorKey, WeakFingerprint, out _)); + } + + [TestMethod] + public void MalformedSelectorsAreRejected() + { + S3ObjectKeys keys = CreateKeys(); + string prefix = keys.GetSelectorPrefix(WeakFingerprint); + + // A listing can contain keys written by a different version of the plugin, so these must not throw. + Assert.IsFalse(keys.TryParseSelector(prefix, WeakFingerprint, out _), "Empty remainder."); + Assert.IsFalse(keys.TryParseSelector($"{prefix}00", WeakFingerprint, out _), "Missing the strong fingerprint."); + Assert.IsFalse(keys.TryParseSelector($"{prefix}not-a-hash/00", WeakFingerprint, out _), "Unparsable PathSet hash."); + Assert.IsFalse(keys.TryParseSelector($"{prefix}a/b/c", WeakFingerprint, out _), "Too many segments."); + + string pathSetHash = keys.GetSelector(CreateStrongFingerprint(new byte[] { 42 })).Substring(prefix.Length).Split('/')[0]; + Assert.IsFalse(keys.TryParseSelector($"{prefix}{pathSetHash}/zz", WeakFingerprint, out _), "Unparsable strong fingerprint."); + } +} diff --git a/src/S3/MSBuildCacheS3Plugin.cs b/src/S3/MSBuildCacheS3Plugin.cs new file mode 100644 index 0000000..f3ab733 --- /dev/null +++ b/src/S3/MSBuildCacheS3Plugin.cs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using BuildXL.Cache.ContentStore.Distributed.NuCache; +using BuildXL.Cache.ContentStore.Hashing; +using BuildXL.Cache.ContentStore.Interfaces.Results; +using BuildXL.Cache.ContentStore.Interfaces.Stores; +using BuildXL.Cache.ContentStore.Interfaces.Tracing; +using BuildXL.Cache.ContentStore.Logging; +using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; +using BuildXL.Cache.MemoizationStore.Sessions; +using Microsoft.Build.Experimental.ProjectCache; +using Microsoft.Build.Framework; +using Microsoft.MSBuildCache.Caching; + +namespace Microsoft.MSBuildCache.S3; + +public sealed class MSBuildCacheS3Plugin : MSBuildCachePluginBase +{ + // Note: Credentials are not in PluginSettings as that's configured through item metadata and thus makes it into + // MSBuild logs. Instead they come from the plugin's constructor or the default AWS credential chain, which + // covers the AWS_* environment variables, the shared credentials file, and instance and task roles. + private readonly AWSCredentials? _credentials; + + // Although S3 is unrelated to Azure DevOps, Vso0 hashing is much faster than SHA256. + protected override HashType HashType => HashType.Vso0; + + // Constructor used when MSBuild creates the plugin + public MSBuildCacheS3Plugin() + { + } + + public MSBuildCacheS3Plugin(AWSCredentials credentials) + { + _credentials = credentials; + } + + protected override async Task CreateCacheClientAsync(PluginLoggerBase logger, CancellationToken cancellationToken) + { + if (Settings == null + || FingerprintFactory == null + || ContentHasher == null + || NugetPackageRoot == null) + { + throw new InvalidOperationException(); + } + + if (string.IsNullOrWhiteSpace(Settings.BucketName)) + { + throw new InvalidOperationException($"{nameof(S3PluginSettings.BucketName)} is required. Set the MSBuildCacheS3BucketName MSBuild property."); + } + + // The cache universe becomes part of the object key, so summarize it with a lowercase hash. +#pragma warning disable CA1308 // S3 object keys are conventionally lowercase + string cacheUniverse = ContentHasher.GetContentHash(Encoding.UTF8.GetBytes(Settings.CacheUniverse)).ToShortString(includeHashType: false).ToLowerInvariant(); +#pragma warning restore CA1308 // S3 object keys are conventionally lowercase + + logger.LogMessage( + $"Using S3 bucket '{Settings.BucketName}'{(Settings.ServiceUrl is null ? null : $" at '{Settings.ServiceUrl}'")} with cache universe '{Settings.CacheUniverse}' as '{cacheUniverse}'.", + MessageImportance.Normal); + + FileLog fileLog = new(Path.Combine(Settings.LogDirectory, "CacheClient.log")); +#pragma warning disable CA2000 // Dispose objects before losing scope. Expected to be disposed using Context.Logger.Dispose in the cache client implementation. + Logger cacheLogger = new(fileLog); +#pragma warning restore CA2000 // Dispose objects before losing scope + Context context = new(cacheLogger); + +#pragma warning disable CA2000 // Dispose objects before losing scope. Expected to be disposed by S3CacheClient + LocalCache localCache = LocalCacheFactory.Create(cacheLogger, Settings.LocalCacheRootPath, Settings.LocalCacheSizeInMegabytes); +#pragma warning restore CA2000 // Dispose objects before losing scope + + ICacheSession localCacheSession = await StartCacheSessionAsync(context, localCache, "local"); + +#pragma warning disable CA2000 // Dispose objects before losing scope. Expected to be disposed by S3CacheClient + IAmazonS3 s3Client = CreateS3Client(Settings); +#pragma warning restore CA2000 // Dispose objects before losing scope + + return new S3CacheClient( + context, + FingerprintFactory, + ContentHasher, + localCache, + localCacheSession, + s3Client, + Settings.BucketName!, + Settings.KeyPrefix, + cacheUniverse, + Settings.RepoRoot, + NugetPackageRoot, + GetFileRealizationMode, + Settings.MaxConcurrentCacheContentOperations, + Settings.RemoteCacheIsReadOnly, + Settings.AsyncCachePublishing, + Settings.AsyncCacheMaterialization, + Settings.SkipUnchangedOutputFiles, + Settings.TouchOutputFiles, + Settings.MultipartThresholdBytes, + Settings.MultipartPartSizeBytes, + Settings.MaxConcurrentPartsPerObject); + } + + private AmazonS3Client CreateS3Client(S3PluginSettings settings) + { + AmazonS3Config config = new() + { + Timeout = TimeSpan.FromMinutes(5), + MaxErrorRetry = 5, + ForcePathStyle = settings.ForcePathStyle, + }; + +#if NETFRAMEWORK + // Size this client's connection pool for the cache operation gate. Note that changing + // ServicePointManager.DefaultConnectionLimit instead would affect unrelated MSBuild tasks. + config.ConnectionLimit = Math.Max(settings.MaxConcurrentCacheContentOperations, settings.MaxConcurrentPartsPerObject); +#endif + + if (settings.ServiceUrl is null) + { + config.RegionEndpoint = RegionEndpoint.GetBySystemName(settings.Region); + } + else + { + config.ServiceURL = settings.ServiceUrl.AbsoluteUri; + config.AuthenticationRegion = settings.Region; + } + + return new AmazonS3Client(_credentials ?? FallbackCredentialsFactory.GetCredentials(), config); + } + + private static async Task StartCacheSessionAsync(Context context, LocalCache cache, string name) + { + await cache.StartupAsync(context).ThrowIfFailure(); + CreateSessionResult cacheSessionResult = cache + .CreateSession(context, name, ImplicitPin.PutAndGet) + .ThrowIfFailure(); + ICacheSession session = cacheSessionResult.Session!; + + (await session.StartupAsync(context)).ThrowIfFailure(); + + return session; + } +} diff --git a/src/S3/Microsoft.MSBuildCache.S3.csproj b/src/S3/Microsoft.MSBuildCache.S3.csproj new file mode 100644 index 0000000..21b5fbd --- /dev/null +++ b/src/S3/Microsoft.MSBuildCache.S3.csproj @@ -0,0 +1,41 @@ + + + + x64 + $(Platform) + net472;net9.0 + + + + + + + + + + + + + + + PreserveNewest + true + build\ + + + PreserveNewest + true + build\ + + + PreserveNewest + true + buildMultiTargeting\ + + + + + + + + diff --git a/src/S3/S3CacheClient.cs b/src/S3/S3CacheClient.cs new file mode 100644 index 0000000..4829052 --- /dev/null +++ b/src/S3/S3CacheClient.cs @@ -0,0 +1,706 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using BuildXL.Cache.ContentStore.Hashing; +using BuildXL.Cache.ContentStore.Interfaces.FileSystem; +using BuildXL.Cache.ContentStore.Interfaces.Results; +using BuildXL.Cache.ContentStore.Interfaces.Sessions; +using BuildXL.Cache.ContentStore.Interfaces.Tracing; +using BuildXL.Cache.ContentStore.UtilitiesCore; +using BuildXL.Cache.MemoizationStore.Interfaces.Caches; +using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; +using Microsoft.MSBuildCache.Caching; +using Microsoft.MSBuildCache.Fingerprinting; +using FileInfo = System.IO.FileInfo; +using Fingerprint = BuildXL.Cache.MemoizationStore.Interfaces.Sessions.Fingerprint; + +namespace Microsoft.MSBuildCache.S3; + +/// +/// Stores the cache in an S3 bucket, or in an S3-compatible store such as MinIO. +/// +/// +/// Unlike there is no BuildXL remote cache to put under a two-level cache, as BuildXL has +/// no S3 implementation. The local cache is only used as a content store and every fingerprint lookup goes to S3. See +/// for the object layout. +/// +internal sealed class S3CacheClient : CacheClient +{ + private const int CopyBufferSize = 128 * 1024; + + // S3 rejects parts smaller than this, except for the last part of a multipart upload. + private const long MinimumPartSizeBytes = 5 * 1024 * 1024; + + private readonly IAmazonS3 _s3; + private readonly IContentHasher _hasher; + private readonly TransferUtility _transferUtility; + private readonly S3ObjectKeys _keys; + private readonly string _bucketName; + private readonly bool _remoteCacheIsReadOnly; + private readonly long _multipartThresholdBytes; + private readonly long _partSizeBytes; + private readonly int _maxConcurrentPartsPerObject; + + // A file copied by many projects is published by each of them, so avoid uploading the same content repeatedly. + private readonly ConcurrentDictionary _remoteUploads = new(); + + // Bounds in-flight part requests across all objects. Without this, each object being downloaded could fan out + // into parts, multiplying the request count by MaxConcurrentCacheContentOperations. + private readonly SemaphoreSlim _partGate; + + public S3CacheClient( + Context rootContext, + IFingerprintFactory fingerprintFactory, + IContentHasher hasher, + ICache localCache, + IContentSession localCas, + IAmazonS3 s3, + string bucketName, + string keyPrefix, + string universe, + string repoRoot, + string nugetPackageRoot, + Func getFileRealizationMode, + int maxConcurrentCacheContentOperations, + bool remoteCacheIsReadOnly, + bool enableAsyncPublishing, + bool enableAsyncMaterialization, + bool skipUnchangedOutputFiles, + bool touchOutputFiles, + long multipartThresholdBytes, + long multipartPartSizeBytes, + int maxConcurrentPartsPerObject) + : base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCas, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization, skipUnchangedOutputFiles, touchOutputFiles) + { + _s3 = s3; + _hasher = hasher; + _bucketName = bucketName; + _remoteCacheIsReadOnly = remoteCacheIsReadOnly; + _multipartThresholdBytes = Math.Max(1, multipartThresholdBytes); + _partSizeBytes = Math.Max(MinimumPartSizeBytes, multipartPartSizeBytes); + _maxConcurrentPartsPerObject = Math.Max(1, maxConcurrentPartsPerObject); + _partGate = new SemaphoreSlim(Math.Max(1, maxConcurrentCacheContentOperations)); + _keys = new S3ObjectKeys(keyPrefix, hasher.Info.HashType, universe); + + _transferUtility = new TransferUtility( + s3, + new TransferUtilityConfig + { + ConcurrentServiceRequests = _maxConcurrentPartsPerObject, + MinSizeBeforePartUpload = _multipartThresholdBytes, + }); + } + + protected override async Task AddNodeAsync( + Context context, + StrongFingerprint fingerprint, + IReadOnlyDictionary outputs, + (ContentHash hash, byte[] bytes) nodeBuildResultBytes, + (ContentHash hash, byte[] bytes)? pathSetBytes, + CancellationToken cancellationToken) + { + if (_remoteCacheIsReadOnly) + { + // S3 is the only fingerprint store, so nothing can be published at all when it's read-only. + return AddNodeResult.Skipped; + } + + string entryKey = _keys.GetEntry(fingerprint); + bool alreadyExists = await ObjectExistsAsync(entryKey, cancellationToken); + + Dictionary contentAbsolutePaths = new(outputs.Count); + foreach (KeyValuePair kvp in outputs) + { + contentAbsolutePaths[kvp.Value] = kvp.Key; + } + + List uploadTasks = new(contentAbsolutePaths.Count + 1); + foreach (KeyValuePair kvp in contentAbsolutePaths) + { + uploadTasks.Add(EnsureContentInRemoteAsync(context, kvp.Key, kvp.Value, cancellationToken)); + } + + // The PathSet is fetched by hash while evaluating selectors, so it needs to be in the CAS. The + // NodeBuildResult does not, as it's stored inline in the entry below. + if (pathSetBytes is not null) + { + uploadTasks.Add(EnsureBytesInRemoteAsync(context, pathSetBytes.Value.hash, pathSetBytes.Value.bytes, cancellationToken)); + } + + await Task.WhenAll(uploadTasks); + + if (!alreadyExists) + { + await PutObjectAsync(entryKey, nodeBuildResultBytes.bytes, cancellationToken); + Tracer.Debug(context, $"Stored S3 cache entry `{entryKey}` for {fingerprint}"); + } + + // Write the selector last so a listed selector always has a readable entry behind it. This is done even when + // the entry already existed, to repair an entry whose selector never landed. + string selectorKey = _keys.GetSelector(fingerprint); + await PutObjectAsync(selectorKey, Array.Empty(), cancellationToken); + Tracer.Debug(context, $"Stored S3 selector `{selectorKey}`"); + + return alreadyExists ? AddNodeResult.AlreadyExists : AddNodeResult.Added; + } + + protected override async Task GetCacheEntryAsync( + Context context, + StrongFingerprint cacheStrongFingerprint, + CancellationToken cancellationToken) + { + string entryKey = _keys.GetEntry(cacheStrongFingerprint); + byte[]? nodeBuildResultBytes = await TryGetObjectBytesAsync(context, entryKey, cancellationToken); + if (nodeBuildResultBytes is null) + { + Tracer.Debug(context, $"S3 cache entry not found for {cacheStrongFingerprint} (`{entryKey}`)"); + return null; + } + + return new CacheEntry(this, nodeBuildResultBytes); + } + + protected override async IAsyncEnumerable GetSelectors( + Context context, + Fingerprint fingerprint, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + string prefix = _keys.GetSelectorPrefix(fingerprint); + Tracer.Debug(context, $"Listing S3 selectors under `{prefix}`"); + + string? continuationToken = null; + do + { + ListObjectsV2Response response; + try + { + response = await _s3.ListObjectsV2Async( + new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = prefix, + ContinuationToken = continuationToken, + }, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // Treat as a weak fingerprint miss; the build still succeeds, it just can't use the cache. + Tracer.Warning(context, $"Failed to list S3 selectors under `{prefix}`: {ex}"); + yield break; + } + + foreach (S3Object s3Object in response.S3Objects) + { + if (_keys.TryParseSelector(s3Object.Key, fingerprint, out Selector selector)) + { + yield return selector; + } + else + { + Tracer.Debug(context, $"Skipping unrecognized S3 selector key `{s3Object.Key}`"); + } + } + + continuationToken = response.IsTruncated ? response.NextContinuationToken : null; + } + while (continuationToken is not null); + } + + /// + /// Only used for PathSets, which the base class fetches while evaluating selectors, so the object is small enough + /// to buffer. Output files are materialized in . + /// + protected override async Task OpenStreamAsync(Context context, ContentHash contentHash, CancellationToken cancellationToken) + { + OpenStreamResult localResult = await LocalCacheSession.OpenStreamAsync(context, contentHash, cancellationToken); + if (localResult.Succeeded) + { + return localResult; + } + + byte[]? bytes = await TryGetObjectBytesAsync(context, _keys.GetCas(contentHash), cancellationToken); + if (bytes is null) + { + return new OpenStreamResult(localResult); + } + + await PutLocalBytesAsync(context, contentHash, bytes, cancellationToken); + return new OpenStreamResult(new MemoryStream(bytes, writable: false)); + } + + public override async ValueTask DisposeAsync() + { + _transferUtility.Dispose(); + _partGate.Dispose(); + _s3.Dispose(); + + await base.DisposeAsync(); + } + + private Task EnsureContentInRemoteAsync( + Context context, + ContentHash contentHash, + string absolutePath, + CancellationToken cancellationToken) + => _remoteUploads.GetOrAdd( + contentHash, + _ => PutOrPlaceFileGate.GatedOperationAsync( + async (_, _) => + { + string casKey = _keys.GetCas(contentHash); + if (await ObjectExistsAsync(casKey, cancellationToken)) + { + return 0; + } + + // With async publishing the content is already in the local cache, so stream from there instead of + // contending with build operations which may still be touching the output. + OpenStreamResult localStream = await LocalCacheSession.OpenStreamAsync(context, contentHash, cancellationToken); + if (localStream.Succeeded && localStream.Stream is not null) + { + using (localStream.Stream) + { + await UploadContentAsync(casKey, localStream.Stream, cancellationToken); + } + + return 0; + } + + if (!File.Exists(absolutePath)) + { + throw new CacheException($"Cannot publish content {contentHash.ToShortString()}: local cache miss and file '{absolutePath}' does not exist."); + } + + using FileStream fileStream = File.OpenRead(absolutePath); + await UploadContentAsync(casKey, fileStream, cancellationToken); + return 0; + }, + cancellationToken)); + + private Task EnsureBytesInRemoteAsync( + Context context, + ContentHash contentHash, + byte[] bytes, + CancellationToken cancellationToken) + => _remoteUploads.GetOrAdd( + contentHash, + _ => PutOrPlaceFileGate.GatedOperationAsync( + async (_, _) => + { + // Also put it locally so OpenStreamAsync can serve it without a round trip. + await PutLocalBytesAsync(context, contentHash, bytes, cancellationToken); + + string casKey = _keys.GetCas(contentHash); + if (!await ObjectExistsAsync(casKey, cancellationToken)) + { + await PutObjectAsync(casKey, bytes, cancellationToken); + } + + return 0; + }, + cancellationToken)); + + private async Task PlaceContentAsync(Context context, ContentHash contentHash, string absolutePath, CancellationToken cancellationToken) + { + CreateParentDirectory(absolutePath); + + PlaceFileResult placeResult = await PlaceFromLocalCacheAsync(context, contentHash, absolutePath, cancellationToken); + if (placeResult.Succeeded) + { + PutLocalTaskCache.TryAdd(contentHash, Task.FromResult(new PutFileOperation(contentHash, BoolResult.Success))); + return; + } + + // Download beside the destination, hand the file to the local cache, then place it from there so the + // realization and access modes end up the same as on a local cache hit. + string tempPath = $"{absolutePath}.msbuildcache-{Guid.NewGuid():N}.tmp"; + try + { + if (!await TryDownloadToFileAsync(context, _keys.GetCas(contentHash), tempPath, contentHash, cancellationToken)) + { + throw new CacheException($"Failed to materialize content {contentHash.ToShortString()} to '{absolutePath}': not found in local or remote cache."); + } + + PutResult putResult = await LocalCacheSession.PutFileAsync( + context, + contentHash, + new AbsolutePath(tempPath), + FileRealizationMode.Move, + cancellationToken); + putResult.ThrowIfFailure(); + PutLocalTaskCache.TryAdd(contentHash, Task.FromResult(new PutFileOperation(contentHash, putResult))); + } + finally + { + // Move consumes the file on success; this covers the failure paths. + TryDeleteFile(tempPath); + } + + (await PlaceFromLocalCacheAsync(context, contentHash, absolutePath, cancellationToken)).ThrowIfFailure(); + } + + private Task PlaceFromLocalCacheAsync(Context context, ContentHash contentHash, string absolutePath, CancellationToken cancellationToken) + { + FileRealizationMode realizationMode = GetFileRealizationMode(absolutePath); + return LocalCacheSession.PlaceFileAsync( + context, + contentHash, + new AbsolutePath(absolutePath), + realizationMode == FileRealizationMode.CopyNoVerify ? FileAccessMode.Write : FileAccessMode.ReadOnly, + FileReplacementMode.ReplaceExisting, + realizationMode, + cancellationToken); + } + + /// + /// Downloads an object to disk without buffering it in memory, splitting it into concurrent ranged requests once + /// it's large enough to be worth the extra round trips. Returns false if the object does not exist. + /// + private async Task TryDownloadToFileAsync( + Context context, + string key, + string absolutePath, + ContentHash expectedContentHash, + CancellationToken cancellationToken) + { + long? length = await TryGetObjectLengthAsync(key, cancellationToken); + if (length is null) + { + return false; + } + + if (length.Value < _multipartThresholdBytes) + { + try + { + using GetObjectResponse response = await _s3.GetObjectAsync(_bucketName, key, cancellationToken); + using FileStream destination = OpenForWrite(absolutePath, FileMode.Create, FileShare.None); + await CopyExactlyAsync(response.ResponseStream, destination, length.Value, key, cancellationToken); + } + catch (AmazonS3Exception ex) when (IsNotFound(ex)) + { + return false; + } + } + else + { + // Establish the full length up front so parts can be written at their own offsets concurrently. + using (FileStream placeholder = OpenForWrite(absolutePath, FileMode.Create, FileShare.ReadWrite)) + { + placeholder.SetLength(length.Value); + } + + // The Task.WhenAll below ensures no part is still holding this when it's disposed. + using SemaphoreSlim objectGate = new(_maxConcurrentPartsPerObject); + + List partTasks = new(); + for (long offset = 0; offset < length.Value; offset += _partSizeBytes) + { + partTasks.Add(DownloadPartAsync(key, absolutePath, offset, Math.Min(offset + _partSizeBytes, length.Value) - 1, objectGate, cancellationToken)); + } + + try + { + await Task.WhenAll(partTasks); + } + catch (AmazonS3Exception ex) when (IsNotFound(ex)) + { + // The object was removed between the metadata call and the ranged reads. + return false; + } + + Tracer.Debug(context, $"Downloaded `{key}` ({length.Value} bytes) as {partTasks.Count} parts"); + } + + long actualLength = new FileInfo(absolutePath).Length; + if (actualLength != length.Value) + { + throw new CacheException($"Downloaded '{key}' to '{absolutePath}' with {actualLength} bytes, expected {length.Value}."); + } + + await VerifyContentHashAsync(key, absolutePath, expectedContentHash); + return true; + } + + /// + /// Parts are bounded twice: caps how much of a single object is in flight, and a + /// client-wide gate caps ranged requests across all objects being downloaded. + /// + private async Task DownloadPartAsync(string key, string absolutePath, long start, long end, SemaphoreSlim objectGate, CancellationToken cancellationToken) + { + await objectGate.WaitAsync(cancellationToken); + try + { + await _partGate.WaitAsync(cancellationToken); + try + { + using GetObjectResponse response = await _s3.GetObjectAsync( + new GetObjectRequest + { + BucketName = _bucketName, + Key = key, + ByteRange = new ByteRange(start, end), + }, + cancellationToken); + + using FileStream destination = OpenForWrite(absolutePath, FileMode.Open, FileShare.ReadWrite); + destination.Seek(start, SeekOrigin.Begin); + await CopyExactlyAsync(response.ResponseStream, destination, end - start + 1, key, cancellationToken); + } + finally + { + _partGate.Release(); + } + } + finally + { + objectGate.Release(); + } + } + + // FileAccess stays qualified as unqualified it binds to the Microsoft.MSBuildCache.FileAccess namespace. + private static FileStream OpenForWrite(string absolutePath, FileMode mode, FileShare share) + => new(absolutePath, mode, System.IO.FileAccess.Write, share, CopyBufferSize, useAsync: true); + + private static async Task CopyExactlyAsync( + Stream source, + Stream destination, + long expectedLength, + string key, + CancellationToken cancellationToken) + { + byte[] buffer = new byte[CopyBufferSize]; + long remaining = expectedLength; + while (remaining > 0) + { + int count = (int)Math.Min(buffer.Length, remaining); +#if NETFRAMEWORK + int read = await source.ReadAsync(buffer, 0, count, cancellationToken); +#else + int read = await source.ReadAsync(buffer.AsMemory(0, count), cancellationToken); +#endif + if (read == 0) + { + throw new CacheException($"S3 object '{key}' ended with {remaining} expected bytes remaining."); + } + +#if NETFRAMEWORK + await destination.WriteAsync(buffer, 0, read, cancellationToken); +#else + await destination.WriteAsync(buffer.AsMemory(0, read), cancellationToken); +#endif + remaining -= read; + } + } + + /// + /// Guards against a truncated or misassembled transfer. A length check alone doesn't catch this, as the file is + /// created at its full length before any part is written. + /// + private async Task VerifyContentHashAsync(string key, string absolutePath, ContentHash expectedContentHash) + { + using FileStream stream = new( + absolutePath, + FileMode.Open, + System.IO.FileAccess.Read, + FileShare.Read, + CopyBufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan); + + ContentHash actualContentHash = await _hasher.GetContentHashAsync(stream); + if (actualContentHash != expectedContentHash) + { + throw new CacheException($"S3 object '{key}' has content hash {actualContentHash}, expected {expectedContentHash}."); + } + } + + private async Task PutLocalBytesAsync(Context context, ContentHash contentHash, byte[] bytes, CancellationToken cancellationToken) + { + if (PutLocalTaskCache.ContainsKey(contentHash)) + { + return; + } + + using MemoryStream stream = new(bytes, writable: false); + PutResult putResult = await LocalCacheSession.PutStreamAsync(context, contentHash.HashType, stream, cancellationToken); + if (!putResult.Succeeded) + { + Tracer.Debug(context, $"Failed to put content {contentHash.ToShortString()} into the local cache: {putResult}"); + return; + } + + if (putResult.ContentHash != contentHash) + { + throw new CacheException($"Content for {contentHash} hashed as {putResult.ContentHash}."); + } + + PutLocalTaskCache.TryAdd(contentHash, Task.FromResult(new PutFileOperation(contentHash, putResult))); + } + + private static void TryDeleteFile(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + + private async Task TryGetObjectBytesAsync(Context context, string key, CancellationToken cancellationToken) + { + try + { + using GetObjectResponse response = await _s3.GetObjectAsync(_bucketName, key, cancellationToken); + using MemoryStream buffer = new(); + await response.ResponseStream.CopyToAsync(buffer +#if !NETFRAMEWORK + , cancellationToken +#endif + ); + return buffer.ToArray(); + } + catch (AmazonS3Exception ex) when (IsNotFound(ex)) + { + return null; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // Treat as a cache miss; the build still succeeds, it just can't use the cache. + Tracer.Warning(context, $"Failed to get S3 object `{key}`: {ex}"); + return null; + } + } + + private async Task TryGetObjectLengthAsync(string key, CancellationToken cancellationToken) + { + try + { + GetObjectMetadataResponse metadata = await _s3.GetObjectMetadataAsync(_bucketName, key, cancellationToken); + return metadata.ContentLength; + } + catch (AmazonS3Exception ex) when (IsNotFound(ex)) + { + return null; + } + } + + private async Task ObjectExistsAsync(string key, CancellationToken cancellationToken) + => await TryGetObjectLengthAsync(key, cancellationToken) is not null; + + /// + /// Uploads cache content, splitting large objects into parts uploaded in parallel. This also caps how much work a + /// failed request throws away, which matters for the very large static libraries some builds produce. + /// + private async Task UploadContentAsync(string key, Stream stream, CancellationToken cancellationToken) + { + // TransferUtility can only split a stream it can seek; a non-seekable one falls back to a single request. + if (!stream.CanSeek || stream.Length < _multipartThresholdBytes) + { + await PutObjectAsync(key, stream, cancellationToken); + return; + } + + await _transferUtility.UploadAsync( + new TransferUtilityUploadRequest + { + BucketName = _bucketName, + Key = key, + InputStream = stream, + PartSize = _partSizeBytes, + AutoCloseStream = false, + }, + cancellationToken); + } + + // Uploads in a single request. The stream stays owned by the caller. + private async Task PutObjectAsync(string key, Stream stream, CancellationToken cancellationToken) + { + _ = await _s3.PutObjectAsync( + new PutObjectRequest + { + BucketName = _bucketName, + Key = key, + InputStream = stream, + AutoCloseStream = false, + }, + cancellationToken); + } + + private async Task PutObjectAsync(string key, byte[] bytes, CancellationToken cancellationToken) + { + using MemoryStream stream = new(bytes, writable: false); + await PutObjectAsync(key, stream, cancellationToken); + } + + internal static bool IsNotFound(AmazonS3Exception ex) + => ex.StatusCode == HttpStatusCode.NotFound + || string.Equals(ex.ErrorCode, "NoSuchKey", StringComparison.OrdinalIgnoreCase) + || string.Equals(ex.ErrorCode, "NoSuchBucket", StringComparison.OrdinalIgnoreCase) + || string.Equals(ex.ErrorCode, "NotFound", StringComparison.OrdinalIgnoreCase); + + private sealed class CacheEntry : ICacheEntry + { + private readonly S3CacheClient _client; + private readonly byte[] _nodeBuildResultBytes; + + public CacheEntry(S3CacheClient client, byte[] nodeBuildResultBytes) + { + _client = client; + _nodeBuildResultBytes = nodeBuildResultBytes; + } + + public void Dispose() + { + } + + public Task GetNodeBuildResultAsync(Context context, CancellationToken cancellationToken) + => Task.FromResult(new MemoryStream(_nodeBuildResultBytes, writable: false)); + + public async Task PlaceFilesAsync(Context context, IReadOnlyDictionary files, CancellationToken cancellationToken) + { + List tasks = new(files.Count); + foreach (KeyValuePair kvp in files) + { + string path = kvp.Key; + ContentHash hash = kvp.Value; + tasks.Add(_client.PutOrPlaceFileGate.GatedOperationAsync( + async (_, _) => + { + await _client.PlaceContentAsync(context, hash, path, cancellationToken); + return 0; + }, + cancellationToken)); + } + + await Task.WhenAll(tasks); + } + } +} diff --git a/src/S3/S3ObjectKeys.cs b/src/S3/S3ObjectKeys.cs new file mode 100644 index 0000000..383c260 --- /dev/null +++ b/src/S3/S3ObjectKeys.cs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using BuildXL.Cache.ContentStore.Hashing; +using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; +using Fingerprint = BuildXL.Cache.MemoizationStore.Interfaces.Sessions.Fingerprint; + +namespace Microsoft.MSBuildCache.S3; + +/// +/// The object key layout of the S3 cache. All keys live under +/// {keyPrefix}/v{layoutVersion}-{hashType}/{universe}/: +/// +/// cas/{contentHash} is content-addressed output files and PathSets. +/// selectors/{weakFingerprint}/{pathSetHash}/{strongFingerprintHex} are empty objects, listed to discover the selectors of a weak fingerprint. +/// entries/{weakFingerprint}/{pathSetHash}/{strongFingerprintHex} is the NodeBuildResult of a strong fingerprint, stored inline rather than by reference since it's never shared between entries. +/// +/// +internal sealed class S3ObjectKeys +{ + // Bumped when a change to the layout or to an object's payload makes existing objects unreadable, so that old and + // new clients don't interpret each other's objects. + private const int LayoutVersion = 1; + + private readonly string _rootPrefix; + + public S3ObjectKeys(string keyPrefix, HashType hashType, string universe) + { + string sanitizedPrefix = keyPrefix.Trim().Trim('/'); + if (string.IsNullOrEmpty(sanitizedPrefix)) + { + sanitizedPrefix = "msbuildcache"; + } + + _rootPrefix = $"{sanitizedPrefix}/v{LayoutVersion}-{(int)hashType}/{universe}"; + } + + public string GetCas(ContentHash contentHash) + => $"{_rootPrefix}/cas/{Escape(contentHash.Serialize())}"; + + public string GetEntry(StrongFingerprint fingerprint) + => $"{_rootPrefix}/entries/{GetSelectorPath(fingerprint.WeakFingerprint, fingerprint.Selector)}"; + + public string GetSelector(StrongFingerprint fingerprint) + => $"{_rootPrefix}/selectors/{GetSelectorPath(fingerprint.WeakFingerprint, fingerprint.Selector)}"; + + public string GetSelectorPrefix(Fingerprint weakFingerprint) + => $"{_rootPrefix}/selectors/{Escape(weakFingerprint.Serialize())}/"; + + /// + /// Reverses . Returns false for any key not produced by this layout, as a listing can + /// contain keys written by a different version of the plugin. + /// + public bool TryParseSelector(string key, Fingerprint weakFingerprint, out Selector selector) + { + selector = default; + + string prefix = GetSelectorPrefix(weakFingerprint); + if (!key.StartsWith(prefix, StringComparison.Ordinal)) + { + return false; + } + + string[] parts = key.Substring(prefix.Length).Split('/'); + if (parts.Length != 2 || !ContentHash.TryParse(Unescape(parts[0]), out ContentHash pathSetHash)) + { + return false; + } + + try + { + selector = new Selector(pathSetHash, HexUtilities.HexToBytes(parts[1])); + return true; + } + catch (ArgumentException) + { + return false; + } + } + + private static string GetSelectorPath(Fingerprint weakFingerprint, Selector selector) + => $"{Escape(weakFingerprint.Serialize())}/{Escape(selector.ContentHash.Serialize())}/{HexUtilities.BytesToHex(selector.Output ?? Array.Empty())}"; + + // Serialized content hashes and fingerprints separate the hash type with ':', which is legal in an S3 key but + // awkward for tooling which maps keys onto file paths. + private static string Escape(string value) => value.Replace(':', '~'); + + private static string Unescape(string value) => value.Replace('~', ':'); +} diff --git a/src/S3/S3PluginSettings.cs b/src/S3/S3PluginSettings.cs new file mode 100644 index 0000000..1a22d09 --- /dev/null +++ b/src/S3/S3PluginSettings.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.MSBuildCache.S3; + +public class S3PluginSettings : PluginSettings +{ + public string? BucketName { get; init; } + + /// + /// The AWS region. Also used as the signing region when is set. + /// + public string Region { get; init; } = "us-east-1"; + + /// + /// The service url of an S3-compatible store such as MinIO. If null, the AWS endpoint for the region is used. + /// + public Uri? ServiceUrl { get; init; } + + /// + /// Whether to use path-style addressing. Most S3-compatible stores require this. + /// + public bool ForcePathStyle { get; init; } + + /// + /// The key prefix under which all cache objects are stored. + /// + public string KeyPrefix { get; init; } = "msbuildcache"; + + /// + /// Objects at or above this size are transferred as multiple parts in parallel instead of as a single request. + /// + /// + /// Build outputs are heavily skewed towards small files, but the few large ones dominate the time to materialize a + /// cache hit with a cold local cache. + /// + public long MultipartThresholdBytes { get; init; } = 32 * 1024 * 1024; + + /// + /// The part size for multipart transfers. Values below S3's 5 MB minimum are raised to it. + /// + /// + /// This also bounds how much of an object is re-transferred when a request fails, as a part is retried on its own + /// rather than restarting the whole object. + /// + public long MultipartPartSizeBytes { get; init; } = 8 * 1024 * 1024; + + /// + /// How many parts of a single object are transferred concurrently. Ranged requests across all multipart transfers + /// are additionally bounded by . + /// + public int MaxConcurrentPartsPerObject { get; init; } = 8; +} diff --git a/src/S3/build/Microsoft.MSBuildCache.S3.props b/src/S3/build/Microsoft.MSBuildCache.S3.props new file mode 100644 index 0000000..6388cdf --- /dev/null +++ b/src/S3/build/Microsoft.MSBuildCache.S3.props @@ -0,0 +1,12 @@ + + + $(MSBuildThisFileDirectory)net472\Microsoft.MSBuildCache.S3.dll + $(MSBuildThisFileDirectory)net9.0\Microsoft.MSBuildCache.S3.dll + + + true + false + + + + diff --git a/src/S3/build/Microsoft.MSBuildCache.S3.targets b/src/S3/build/Microsoft.MSBuildCache.S3.targets new file mode 100644 index 0000000..99bb0b0 --- /dev/null +++ b/src/S3/build/Microsoft.MSBuildCache.S3.targets @@ -0,0 +1,27 @@ + + + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheS3BucketName + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheS3Region + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheS3ServiceUrl + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheS3ForcePathStyle + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheS3KeyPrefix + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheS3MultipartThresholdBytes + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheS3MultipartPartSizeBytes + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheS3MaxConcurrentPartsPerObject + + + + + + + $(MSBuildCacheS3BucketName) + $(MSBuildCacheS3Region) + $(MSBuildCacheS3ServiceUrl) + $(MSBuildCacheS3ForcePathStyle) + $(MSBuildCacheS3KeyPrefix) + $(MSBuildCacheS3MultipartThresholdBytes) + $(MSBuildCacheS3MultipartPartSizeBytes) + $(MSBuildCacheS3MaxConcurrentPartsPerObject) + + + diff --git a/src/S3/buildMultitargeting/Microsoft.MSBuildCache.S3.props b/src/S3/buildMultitargeting/Microsoft.MSBuildCache.S3.props new file mode 100644 index 0000000..c316cc4 --- /dev/null +++ b/src/S3/buildMultitargeting/Microsoft.MSBuildCache.S3.props @@ -0,0 +1,3 @@ + + + diff --git a/src/S3/buildMultitargeting/Microsoft.MSBuildCache.S3.targets b/src/S3/buildMultitargeting/Microsoft.MSBuildCache.S3.targets new file mode 100644 index 0000000..c316cc4 --- /dev/null +++ b/src/S3/buildMultitargeting/Microsoft.MSBuildCache.S3.targets @@ -0,0 +1,3 @@ + + + diff --git a/tests/TestProject/TestProject.csproj b/tests/TestProject/TestProject.csproj index e8bb630..77498c1 100644 --- a/tests/TestProject/TestProject.csproj +++ b/tests/TestProject/TestProject.csproj @@ -15,4 +15,39 @@ + + + + + + + + + + + + + + + + + diff --git a/tests/lib.ps1 b/tests/lib.ps1 index c68ee9f..4bed942 100644 --- a/tests/lib.ps1 +++ b/tests/lib.ps1 @@ -108,7 +108,8 @@ function Invoke-MSBuildCacheBuild $output = Get-Content $stdout -Raw $hitMatch = [regex]::Match($output, 'Cache Hit Count: (?\d+)') $missMatch = [regex]::Match($output, 'Cache Miss Count: (?\d+)') - $ratioMatch = [regex]::Match($output, 'Cache Hit Ratio: (?\d+\.\d+%)') + # The ratio is formatted for the build's culture, so the decimal separator and the spacing before '%' vary. + $ratioMatch = [regex]::Match($output, 'Cache Hit Ratio: (?\d+[.,]\d+\s*%)') if (-not ($hitMatch.Success -and $missMatch.Success -and $ratioMatch.Success)) { throw "[$Context] could not parse cache statistics from $stdout." @@ -122,6 +123,16 @@ function Invoke-MSBuildCacheBuild } } +function Get-ComparableRatio +{ + param( + [Parameter(Mandatory = $true)] + [string] $Ratio + ) + + return ($Ratio -replace '\s', '') -replace ',', '.' +} + function Assert-CacheStats { param( @@ -150,9 +161,12 @@ function Assert-CacheStats } $expectedRatio = "{0:P1}" -f ($ExpectedHits / ($ExpectedHits + $ExpectedMisses)) + + # The build and this script may format the ratio differently, since a percentage's decimal separator and the + # spacing before '%' are culture specific, and the space can be non-breaking. $matches = $Result.Hits -eq $ExpectedHits ` -and $Result.Misses -eq $ExpectedMisses ` - -and $Result.HitRatio -eq $expectedRatio + -and (Get-ComparableRatio $Result.HitRatio) -eq (Get-ComparableRatio $expectedRatio) $marker = if ($matches) { "PASS" } else { "FAIL" } Write-Host (" [{0,4}] {1} hits={2} misses={3} ratio={4} (expected hits={5} misses={6} ratio={7})" ` diff --git a/tests/s3.ps1 b/tests/s3.ps1 new file mode 100644 index 0000000..5f28c68 --- /dev/null +++ b/tests/s3.ps1 @@ -0,0 +1,338 @@ +# tests/s3.ps1 +# +# End-to-end test for the Microsoft.MSBuildCache.S3 plugin. It runs the same cold and warm cache builds as +# smoke.ps1, plus a build with the local cache deleted so cache hits have to be served from S3. It also produces +# one output large enough to take the multipart transfer path and verifies it round trips byte for byte. +# +# By default a local moto server is used as the S3 endpoint. moto is a pure-Python AWS mock, so unlike MinIO it +# needs no container runtime, which makes it usable on a Windows build agent. Pass -S3ServiceUrl to test against an +# existing S3-compatible endpoint instead. +# +# To run: +# .\tests\s3.ps1 + +param +( + [Parameter(Mandatory = $false)] + [string] $LogDirectory = $env:LogDirectory, + + [Parameter(Mandatory = $false)] + [string] $LocalPackageDir = $env:LocalPackageDir, + + [Parameter(Mandatory = $false)] + [string] $TestRoot, + + [Parameter(Mandatory = $false)] + [string] $MSBuildPath = $null, + + [Parameter(Mandatory = $false)] + [string] $Configuration = "Debug", + + [Parameter(Mandatory = $false)] + [string] $BucketName = "msbuildcache-test", + + # An existing S3-compatible endpoint. When omitted, a local moto server is started and the bucket is created in it. + [Parameter(Mandatory = $false)] + [string] $S3ServiceUrl, + + [Parameter(Mandatory = $false)] + [int] $S3Port = 9099, + + # Size of the additional output used to cover multipart transfers. 0 disables it. + [Parameter(Mandatory = $false)] + [int] $LargeOutputMegabytes = 24 +) + +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" +. (Join-Path $PSScriptRoot "lib.ps1") + +$CachePackage = "Microsoft.MSBuildCache.S3" + +# Recorded on the first build and compared on every later one. +$Script:ExpectedLargeOutputHash = $null + +function Test-PortListening +{ + param( + [Parameter(Mandatory = $true)] + [int] $Port + ) + + $client = New-Object System.Net.Sockets.TcpClient + try + { + $client.Connect("127.0.0.1", $Port) + return $true + } + catch + { + return $false + } + finally + { + $client.Dispose() + } +} + +function Start-MotoServer +{ + param( + [Parameter(Mandatory = $true)] + [int] $Port, + + [Parameter(Mandatory = $true)] + [string] $LogDirectory + ) + + if (Test-PortListening -Port $Port) + { + throw "Port $Port is already in use. Pass -S3Port to select another port, or -S3ServiceUrl to use an existing endpoint." + } + + # Prefer uv, which runs moto from an ephemeral environment without installing anything. + if (Get-Command "uv" -ErrorAction SilentlyContinue) + { + $filePath = "uv" + $argumentList = @("run", "--no-project", "--with", "moto[s3,server]==5.2.2", "moto_server", "-p", "$Port") + } + elseif (Get-Command "moto_server" -ErrorAction SilentlyContinue) + { + $filePath = "moto_server" + $argumentList = @("-p", "$Port") + } + else + { + throw "Could not find 'uv' or 'moto_server'. Install uv (https://docs.astral.sh/uv), or run 'pip install moto[s3,server]', or pass -S3ServiceUrl to use an existing endpoint." + } + + Write-Host "Starting S3 server: $filePath $argumentList" + $process = Start-Process -FilePath $filePath -ArgumentList $argumentList ` + -RedirectStandardOutput (Join-Path $LogDirectory "moto-stdout.txt") ` + -RedirectStandardError (Join-Path $LogDirectory "moto-stderr.txt") ` + -PassThru -NoNewWindow + + $deadline = (Get-Date).AddSeconds(120) + while (-not (Test-PortListening -Port $Port)) + { + if ($process.HasExited) + { + throw "S3 server exited with code $($process.ExitCode). See $LogDirectory\moto-stderr.txt." + } + + if ((Get-Date) -gt $deadline) + { + throw "S3 server did not start listening on port $Port in time. See $LogDirectory\moto-stderr.txt." + } + + Start-Sleep -Milliseconds 500 + } + + Write-Host "S3 server listening on port $Port (pid $($process.Id))" + return $process +} + +function Assert-LargeOutput +{ + param( + [Parameter(Mandatory = $true)] + [string] $ProjectDir, + + [Parameter(Mandatory = $true)] + [string] $Context + ) + + $largeOutput = Get-ChildItem -Path $ProjectDir -Recurse -File -Filter "large.bin" | Select-Object -First 1 + if (-not $largeOutput) + { + throw "[$Context] the large output was not produced." + } + + $expectedSize = $LargeOutputMegabytes * 1MB + if ($largeOutput.Length -ne $expectedSize) + { + throw "[$Context] the large output has $($largeOutput.Length) bytes, expected $expectedSize." + } + + # A transfer assembled from parts can produce a file of the right length but the wrong content, so compare against + # what the build originally produced. + $hash = (Get-FileHash -Path $largeOutput.FullName -Algorithm SHA256).Hash + if (-not $Script:ExpectedLargeOutputHash) + { + $Script:ExpectedLargeOutputHash = $hash + } + elseif ($hash -ne $Script:ExpectedLargeOutputHash) + { + throw "[$Context] the large output was restored with different content. Expected SHA256 $($Script:ExpectedLargeOutputHash), actual $hash." + } + + Write-Host " [PASS] $Context large output verified ($($largeOutput.Length) bytes, SHA256 $hash)" +} + +function Run-Test +{ + param( + [Parameter(Mandatory = $true)] + [string] $TestName, + + [Parameter(Mandatory = $true)] + [int] $ExpectedCacheHits, + + [Parameter(Mandatory = $true)] + [int] $ExpectedCacheMisses, + + # Deletes the local cache before building, so that cache hits have to come from S3. + [Parameter(Mandatory = $false)] + [switch] $ClearLocalCache + ) + + Write-Host "[$TestName] Starting test" + + Write-Host "[$TestName] Cleaning" + Push-Location $ProjectDir + & git clean -fdx + Pop-Location + + if ($ClearLocalCache) + { + Write-Host "[$TestName] Clearing the local cache" + Remove-Item -Path $CacheRoot -Recurse -Force -ErrorAction SilentlyContinue + } + + $extraProperties = @{ + "MSBuildCacheS3BucketName" = $BucketName + "MSBuildCacheS3ServiceUrl" = $S3ServiceUrl + "MSBuildCacheS3ForcePathStyle" = "true" + } + + if ($LargeOutputMegabytes -gt 0) + { + $extraProperties["LargeOutputSizeInMegabytes"] = $LargeOutputMegabytes + + # Put the large output on the multipart path while leaving the small outputs on the single-request path, so + # that one run covers both. + $extraProperties["MSBuildCacheS3MultipartThresholdBytes"] = 8MB + $extraProperties["MSBuildCacheS3MultipartPartSizeBytes"] = 5MB + } + + Write-Host "[$TestName] Building" + $result = Invoke-MSBuildCacheBuild ` + -MSBuildPath $MSBuildPath ` + -ProjectDir $ProjectDir ` + -LogDirectory (Join-Path $LogDirectory $TestName) ` + -CachePackage $CachePackage ` + -CacheUniverse $CacheUniverse ` + -CacheRoot $CacheRoot ` + -ExtraProperties $extraProperties ` + -Context $TestName + + Assert-CacheStats ` + -Result $result ` + -ExpectedHits $ExpectedCacheHits ` + -ExpectedMisses $ExpectedCacheMisses ` + -Context $TestName + + if ($LargeOutputMegabytes -gt 0) + { + Assert-LargeOutput -ProjectDir $ProjectDir -Context $TestName + } + + Write-Host "[$TestName] Test complete" +} + +Push-Location (Join-Path $PSScriptRoot "..") +$RepoRoot = "$PWD" +Pop-Location + +if (-not $LocalPackageDir) +{ + $LocalPackageDir = Join-Path $RepoRoot "artifacts\$Configuration\packages" +} + +if (-not $LogDirectory) +{ + $LogDirectory = Join-Path $RepoRoot "logs\Tests" +} + +if (-not $TestRoot) +{ + $TestRoot = Join-Path $RepoRoot "TestResult\$CachePackage" +} + +if (-not $MSBuildPath) +{ + # Find it on the PATH + $MSBuildPath = (Get-Command "msbuild").Path +} + +# Use a unique cache universe for every test run +$CacheUniverse = (New-Guid).ToString() +$CacheRoot = Join-Path $TestRoot "MSBuildCache" + +$env:LocalPackageDir = $LocalPackageDir + +Write-Host "Log Directory: $LogDirectory" +Remove-Item -Path $LogDirectory -Recurse -Force -ErrorAction SilentlyContinue +New-Item -ItemType Directory -Path $LogDirectory > $null + +# The AWS SDK requires credentials in order to sign requests, even against an endpoint which does not verify them. +if (-not $env:AWS_ACCESS_KEY_ID) +{ + $env:AWS_ACCESS_KEY_ID = "testonly" + $env:AWS_SECRET_ACCESS_KEY = "testonly" +} + +$MotoServer = $null +try +{ + if (-not $S3ServiceUrl) + { + $MotoServer = Start-MotoServer -Port $S3Port -LogDirectory $LogDirectory + $S3ServiceUrl = "http://127.0.0.1:$S3Port" + + # moto does not verify signatures, so an unauthenticated create is enough. An existing endpoint passed via + # -S3ServiceUrl is expected to already have the bucket. + Invoke-WebRequest -Method Put -Uri "$S3ServiceUrl/$BucketName" -UseBasicParsing > $null + Write-Host "Created bucket $BucketName" + } + + Write-Host "Using S3 endpoint $S3ServiceUrl and bucket $BucketName" + Write-Host "Running test in $TestRoot" + + $env:NUGET_PACKAGES = "$TestRoot\.nuget" + $ProjectDir = Join-Path $TestRoot "src" + + Remove-Item -Path $TestRoot -Recurse -Force -ErrorAction SilentlyContinue + + Write-Host "Creating Git repo in $ProjectDir" + New-MSBuildCacheTestProject ` + -ProjectDir $ProjectDir ` + -GitUserName $Env:UserName ` + -GitUserEmail "$Env:UserName@microsoft.com" + + Run-Test ` + -TestName "ColdCache" ` + -ExpectedCacheHits 0 ` + -ExpectedCacheMisses 1 + + Run-Test ` + -TestName "WarmCache" ` + -ExpectedCacheHits 1 ` + -ExpectedCacheMisses 0 + + # Without the local cache, a hit proves the metadata and content round tripped through S3. + Run-Test ` + -TestName "WarmCacheRemoteOnly" ` + -ExpectedCacheHits 1 ` + -ExpectedCacheMisses 0 ` + -ClearLocalCache +} +finally +{ + if ($MotoServer -and -not $MotoServer.HasExited) + { + Write-Host "Stopping S3 server (pid $($MotoServer.Id))" + $MotoServer.Kill($true) + $MotoServer.WaitForExit() + } +}