diff --git a/CLAUDE.md b/CLAUDE.md index 0ad8c3e..cdd0dae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -85,18 +85,22 @@ Configuration is read from the `"CloudFiles"` section in `appsettings.json` (`Cl ### Automatic Lifecycle Rules -Three providers automatically create delete rules for temp-prefix objects at DI registration time. All three check for existing rules and only add missing ones. +Every provider creates delete rules for temp-prefix objects at DI registration time, checks for existing rules and only adds missing (or disabled) ones, and writes every other rule back unchanged. Each also registers `ICloudFilesLifecycle` (PrimitiveTypes), which reads the rules back; the per-provider mapping lives in an internal static `*LifecycleRules` class with pure unit tests. | Provider | Mechanism | Opt-out | |----------|-----------|---------| | S3 | S3 Lifecycle Configuration API | `S3CloudFilesOptions.DisableAutoLifecycle = true` | | GCS | GCS Bucket Lifecycle API (`PatchBucket`) | `GoogleCloudFilesOptions.DisableAutoLifecycle = true` | | Oracle | OCI Object Lifecycle Policy API (`PutObjectLifecyclePolicy`) | `OracleCloudFilesOptions.DisableAutoLifecycle = true` | -| Azure | **Not automatic** — ARM plane required | N/A (property exists but is no-op) | +| Azure | Account lifecycle management policy via ARM (`Azure.ResourceManager.Storage` 1.6.2), only when `SubscriptionId` + `ResourceGroupName` are set | `AzureCloudFilesOptions.DisableAutoLifecycle = true` | | LocalTests | N/A — call `CloudFilesService.Cleanup()` in test teardown | N/A | Lifecycle prefixes: `temp1/` (1 day), `temp7/` (7 days), `temp30/` (30 days), `temp365/` (365 days). +The S3 lifecycle API replaces the whole configuration on every write; before 10.0.1 `AddS3CloudFiles` sent only the missing temp rules and so deleted every other rule on the bucket. `S3LifecycleRules.WithTempRules` now returns the full merged set. + +`Azure.ResourceManager.Storage` is pinned to 1.6.2: 1.7+ needs an Azure.Core in which the Identity types moved into Core, which clashes with `Azure.Identity` 1.13 (CS0433) here and in every consuming app. + S3 and GCS also auto-create the bucket if it does not exist. ### GCS: `StorageClient` and `UrlSigner` via DI diff --git a/README.md b/README.md index 90c4156..bfc4bd9 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ public class FilesController : ControllerBase ## Automatic Lifecycle Management -Three of the four providers automatically create **delete lifecycle rules** for temp-prefix objects on startup. This is useful for objects that are only needed temporarily. +Every provider creates **delete lifecycle rules** for temp-prefix objects on startup — Azure once it can reach the storage account's policy (see below). This is useful for objects that are only needed temporarily. | Prefix | Expiry | |--------|--------| @@ -276,7 +276,18 @@ Three of the four providers automatically create **delete lifecycle rules** for | `temp30/` | 30 days | | `temp365/` | 365 days | -Rules are only added if they don't already exist — existing rules are never modified or removed. +Rules are only added if they don't already exist (a disabled temp rule is switched back on). Every other rule on the bucket is kept as it is. Before 10.0.1 the S3 provider wrote only the missing temp rules, which replaced the bucket's whole configuration and deleted any other rules on it. + +### Reading the rules back + +Each provider also registers `ICloudFilesLifecycle` (from `SimplyWorks.PrimitiveTypes`), which reports the bucket's deletion rules as the storage service has them — prefix, days and whether each is enabled — so an app can say how long a file under a given key is kept: + +```csharp +var lifecycle = await serviceProvider.GetRequiredService().GetLifecycleAsync(); +var rule = lifecycle.RuleFor("temp30/invoices/123.pdf"); // null when nothing deletes it +``` + +Only rules that delete by age are reported; rules that move objects to another tier, delete on a date, or only reach tagged objects are left out. `Unavailable` explains when the rules can't be known without more configuration (Azure without its policy settings). ### Disabling Lifecycle Management @@ -297,6 +308,11 @@ services.AddOracleCloudFiles(o => { // ... other config o.DisableAutoLifecycle = true; }); + +services.AddAsCloudFiles(o => { + // ... other config + o.DisableAutoLifecycle = true; +}); ``` ### Provider Support @@ -306,7 +322,7 @@ services.AddOracleCloudFiles(o => { | S3 | ✅ Automatic | Rules applied via S3 Lifecycle Configuration API | | Google Cloud | ✅ Automatic | Rules applied via GCS Bucket Lifecycle API; bucket is created if it doesn't exist | | Oracle Cloud | ✅ Automatic | Rules applied via OCI Object Lifecycle Policy API | -| Azure Blob | ⚠️ Manual | Azure lifecycle management requires the Azure Resource Manager (ARM) plane, which is separate from the data-plane SDK used by this library. Configure lifecycle rules via the [Azure Portal](https://learn.microsoft.com/en-us/azure/storage/blobs/lifecycle-management-overview), Azure CLI (`az storage account management-policy create`), or ARM templates. | +| Azure Blob | ✅ Automatic, once configured | The rules live in the storage account's [lifecycle management policy](https://learn.microsoft.com/en-us/azure/storage/blobs/lifecycle-management-overview), on the Azure Resource Manager plane. Set `SubscriptionId` and `ResourceGroupName` (and `StorageAccountName` when the service URL is a custom domain or private link). The managed identity, or a service principal in `AZURE_TENANT_ID` / `AZURE_CLIENT_ID` / `AZURE_CLIENT_SECRET`, needs `Microsoft.Storage/storageAccounts/managementPolicies/read` and `write` — Storage Account Contributor has them; Storage Blob Data Contributor doesn't. Rules are named `{container}-temp30` and filtered to `{container}/temp30/`, so other containers' rules are untouched. Changes can take up to 24 hours to apply. Without the two settings nothing is created and nothing deletes temp files. | ## Signed URLs diff --git a/SW.CloudFiles.AS.Extensions/IServiceCollectionExtensions.cs b/SW.CloudFiles.AS.Extensions/IServiceCollectionExtensions.cs index 021f95e..bf625d6 100644 --- a/SW.CloudFiles.AS.Extensions/IServiceCollectionExtensions.cs +++ b/SW.CloudFiles.AS.Extensions/IServiceCollectionExtensions.cs @@ -1,4 +1,6 @@ using System; +using Azure; +using Azure.ResourceManager.Storage; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using SW.CloudFiles.AS; @@ -10,13 +12,16 @@ namespace SW.CloudFiles.AS.Extensions; public static class IServiceCollectionExtensions { /// - /// Registers Azure Blob Storage as the implementation. + /// Registers Azure Blob Storage as the implementation, and + /// , which reads the storage account's deletion rules back. /// On startup, creates the container if it does not exist. /// - /// Note: Azure lifecycle management policies require the Azure Resource Manager (ARM) plane and - /// are not configured automatically. Set them via the Azure Portal, Azure CLI - /// (az storage account management-policy create), or ARM templates. - /// is accepted but has no effect. + /// Once and + /// are set, and unless + /// is true, also ensures the account's lifecycle + /// management policy deletes this container's blobs under temp1/, temp7/, temp30/ and temp365/ after 1, 7, + /// 30 and 365 days, keeping every other rule in the policy. The policy lives on the Azure Resource Manager + /// plane, so the identity needs a role that can manage it; see . /// /// public static IServiceCollection AddAsCloudFiles(this IServiceCollection serviceCollection, Action configure = null) @@ -30,10 +35,34 @@ public static IServiceCollection AddAsCloudFiles(this IServiceCollection service var blobContainerClient = cloudFilesOptions.CreateClient(); + if (!cloudFilesOptions.DisableAutoLifecycle && cloudFilesOptions.CanManage()) + EnsureLifecycleRules(cloudFilesOptions); + serviceCollection.AddSingleton(blobContainerClient); serviceCollection.AddSingleton(cloudFilesOptions); serviceCollection.AddSingleton(cloudFilesOptions); - serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); return serviceCollection; } + + private static void EnsureLifecycleRules(AzureCloudFilesOptions options) + { + var policy = options.ManagementPolicy(); + + StorageAccountManagementPolicyData data; + try + { + data = policy.Get().Value.Data; + } + catch (RequestFailedException ex) when (ex.Status == 404 && ex.ErrorCode == "ManagementPolicyNotFound") + { + data = new StorageAccountManagementPolicyData(); + } + + // The policy is written whole, so everything already in it goes back with the additions. + if (AzureLifecycleRules.AddTempRules(data, options.BucketName)) + policy.CreateOrUpdate(WaitUntil.Completed, data); + } } \ No newline at end of file diff --git a/SW.CloudFiles.AS.UnitTest/AzureLifecycleRulesTests.cs b/SW.CloudFiles.AS.UnitTest/AzureLifecycleRulesTests.cs new file mode 100644 index 0000000..0ab17ca --- /dev/null +++ b/SW.CloudFiles.AS.UnitTest/AzureLifecycleRulesTests.cs @@ -0,0 +1,101 @@ +using System.Collections.Generic; +using System.Linq; +using Azure.ResourceManager.Storage; +using Azure.ResourceManager.Storage.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.CloudFiles.AS; + +namespace SW.CloudFiles.AS.UnitTest; + +/// Pure policy logic, no storage account needed — unlike the other tests in this project. +[TestClass] +public class AzureLifecycleRulesTests +{ + private static ManagementPolicyRule DeleteRule(string name, float days, params string[] prefixes) + { + var filter = new ManagementPolicyFilter(["blockBlob"]); + foreach (var prefix in prefixes) filter.PrefixMatch.Add(prefix); + var actions = new ManagementPolicyAction + { + BaseBlob = new ManagementPolicyBaseBlob { Delete = new DateAfterModification { DaysAfterModificationGreaterThan = days } } + }; + return new ManagementPolicyRule(name, ManagementPolicyRuleType.Lifecycle, + new ManagementPolicyDefinition(actions) { Filters = filter }); + } + + [TestMethod] + public void Adds_this_containers_temp_rules_and_keeps_the_rest_of_the_policy() + { + var data = new StorageAccountManagementPolicyData(); + data.Rules.Add(DeleteRule("someone-elses", 14, "logs/")); + data.Rules.Add(DeleteRule("other-temp30", 30, "other/temp30/")); + + Assert.IsTrue(AzureLifecycleRules.AddTempRules(data, "docs")); + + CollectionAssert.AreEquivalent( + new[] { "someone-elses", "other-temp30", "docs-temp1", "docs-temp7", "docs-temp30", "docs-temp365" }, + data.Rules.Select(r => r.Name).ToArray()); + Assert.AreEqual("docs/temp30/", data.Rules.Single(r => r.Name == "docs-temp30").Definition.Filters.PrefixMatch.Single()); + } + + [TestMethod] + public void Nothing_to_write_when_the_rules_are_already_there() + { + var data = new StorageAccountManagementPolicyData(); + AzureLifecycleRules.AddTempRules(data, "docs"); + + Assert.IsFalse(AzureLifecycleRules.AddTempRules(data, "docs")); + } + + [TestMethod] + public void A_disabled_temp_rule_is_replaced_not_duplicated() + { + var data = new StorageAccountManagementPolicyData(); + var disabled = DeleteRule("docs-temp30", 30, "docs/temp30/"); + disabled.IsEnabled = false; + data.Rules.Add(disabled); + + AzureLifecycleRules.AddTempRules(data, "docs"); + + var temp30 = data.Rules.Where(r => r.Name == "docs-temp30").ToList(); + Assert.AreEqual(1, temp30.Count); + Assert.IsTrue(temp30[0].IsEnabled); + } + + [TestMethod] + public void Reads_only_this_containers_prefixes_relative_to_the_container() + { + var rules = AzureLifecycleRules.ToDeletionRules([ + DeleteRule("docs-temp30", 30, "docs/temp30/"), + DeleteRule("other-temp30", 30, "other/temp30/") + ], "docs"); + + var rule = rules.Single(); + Assert.AreEqual("temp30/", rule.Prefix); + Assert.AreEqual(30, rule.Days); + Assert.IsTrue(rule.Enabled); + } + + [TestMethod] + public void A_rule_without_prefixes_covers_every_container() + { + Assert.AreEqual(string.Empty, AzureLifecycleRules.ToDeletionRules([DeleteRule("all", 90)], "docs").Single().Prefix); + } + + [TestMethod] + public void Tag_filtered_rules_and_rules_without_a_delete_are_left_out() + { + var tagged = DeleteRule("tagged", 7, "docs/temp7/"); + tagged.Definition.Filters.BlobIndexMatch.Add(new ManagementPolicyTagFilter("kind", "==", "scratch")); + var tiering = new ManagementPolicyRule("tiering", ManagementPolicyRuleType.Lifecycle, + new ManagementPolicyDefinition(new ManagementPolicyAction + { + BaseBlob = new ManagementPolicyBaseBlob + { + TierToCool = new DateAfterModification { DaysAfterModificationGreaterThan = 30 } + } + })); + + Assert.AreEqual(0, AzureLifecycleRules.ToDeletionRules([tagged, tiering], "docs").Count); + } +} diff --git a/SW.CloudFiles.AS/AzureCloudFilesOptions.cs b/SW.CloudFiles.AS/AzureCloudFilesOptions.cs index bcf828a..4aa05d4 100644 --- a/SW.CloudFiles.AS/AzureCloudFilesOptions.cs +++ b/SW.CloudFiles.AS/AzureCloudFilesOptions.cs @@ -27,10 +27,29 @@ public class AzureCloudFilesOptions : CloudFilesOptions public string PublicServiceUrl { get; set; } /// - /// Reserved for future use. Azure Blob Storage lifecycle management policies require the - /// Azure Resource Manager (ARM) plane and cannot be configured automatically through - /// the data-plane SDK. Configure lifecycle rules via the Azure Portal, Azure CLI - /// (az storage account management-policy create), or ARM templates. + /// When true, skips creating the temp-prefix delete rules (temp1/, temp7/, temp30/, temp365/) in the + /// storage account's lifecycle management policy. Only applies once and + /// are set: the policy lives on the Azure Resource Manager plane, not + /// the blob endpoint, so without them nothing can be created and nothing deletes temp files. /// public bool DisableAutoLifecycle { get; set; } + + /// + /// Subscription of the storage account, for its lifecycle management policy. Leave unset and the + /// policy is neither created nor read. The identity used — the managed identity, or a service principal + /// from AZURE_TENANT_ID / AZURE_CLIENT_ID / AZURE_CLIENT_SECRET — needs + /// Microsoft.Storage/storageAccounts/managementPolicies/read and /write, for example the + /// Storage Account Contributor role; Storage Blob Data Contributor alone isn't enough. + /// + public string SubscriptionId { get; set; } + + /// Resource group of the storage account. Required alongside . + public string ResourceGroupName { get; set; } + + /// + /// The storage account's name. Optional: taken from with a + /// shared key, or from the first part of the service URL's host with a managed identity. Set it when + /// the service URL is a custom domain or a private link. + /// + public string StorageAccountName { get; set; } } \ No newline at end of file diff --git a/SW.CloudFiles.AS/AzureLifecycleRules.cs b/SW.CloudFiles.AS/AzureLifecycleRules.cs new file mode 100644 index 0000000..97b4066 --- /dev/null +++ b/SW.CloudFiles.AS/AzureLifecycleRules.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Azure.Identity; +using Azure.ResourceManager; +using Azure.ResourceManager.Storage; +using Azure.ResourceManager.Storage.Models; +using SW.PrimitiveTypes; + +namespace SW.CloudFiles.AS; + +/// +/// The storage account's lifecycle management policy: the temp-prefix rules this library keeps in it, and +/// how its rules read as deletion rules for one container. +/// +/// +/// The policy belongs to the whole account and applies by container/prefix, so every rule this +/// library writes is named and filtered for its own container, and the rules of other containers — or +/// anyone else's — are kept as they are. The policy is always read and written whole. +/// +internal static class AzureLifecycleRules +{ + internal static readonly (string Id, string Prefix, int Days)[] Temp = + [ + ("temp1", "temp1/", 1), + ("temp7", "temp7/", 7), + ("temp30", "temp30/", 30), + ("temp365", "temp365/", 365) + ]; + + /// Whether the account's policy can be reached: it lives on the Resource Manager plane, found by subscription and resource group. + internal static bool CanManage(this AzureCloudFilesOptions options) => + !string.IsNullOrWhiteSpace(options.SubscriptionId) && !string.IsNullOrWhiteSpace(options.ResourceGroupName); + + internal static string AccountName(this AzureCloudFilesOptions options) => + !string.IsNullOrWhiteSpace(options.StorageAccountName) ? options.StorageAccountName + : options.Managed ? new Uri(options.ServiceUrl).Host.Split('.')[0] + : options.AccessKeyId; + + /// + /// The account's management policy. The managed identity when one is configured, and otherwise + /// whatever finds — a service principal in the AZURE_* variables. + /// A shared key can't be used: it only opens the blob endpoint. + /// + internal static StorageAccountManagementPolicyResource ManagementPolicy(this AzureCloudFilesOptions options) + { + var credential = options.ManagedIdentityClientId is not null + ? new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = options.ManagedIdentityClientId }) + : new DefaultAzureCredential(); + var account = StorageAccountResource.CreateResourceIdentifier(options.SubscriptionId, options.ResourceGroupName, + options.AccountName()); + return new ArmClient(credential).GetStorageAccountResource(account).GetStorageAccountManagementPolicy(); + } + + internal static string RuleName(string container, string id) => $"{container}-{id}"; + + /// + /// Adds each missing or disabled temp rule for to . + /// Returns whether anything changed, i.e. whether the policy needs writing back. + /// + internal static bool AddTempRules(StorageAccountManagementPolicyData data, string container) + { + var changed = false; + foreach (var (id, prefix, days) in Temp) + { + var name = RuleName(container, id); + if (data.Rules.Any(r => r.Name == name && r.IsEnabled != false)) continue; + + foreach (var stale in data.Rules.Where(r => r.Name == name).ToList()) data.Rules.Remove(stale); + + var filter = new ManagementPolicyFilter(["blockBlob"]); + filter.PrefixMatch.Add($"{container}/{prefix}"); + var actions = new ManagementPolicyAction + { + BaseBlob = new ManagementPolicyBaseBlob + { + Delete = new DateAfterModification { DaysAfterModificationGreaterThan = days } + } + }; + data.Rules.Add(new ManagementPolicyRule(name, ManagementPolicyRuleType.Lifecycle, + new ManagementPolicyDefinition(actions) { Filters = filter }) { IsEnabled = true }); + changed = true; + } + + return changed; + } + + /// + /// The rules that delete 's block blobs by age, with prefixes relative to the + /// container. A rule filtered on blob index tags, or counting from last access, only reaches some of the + /// blobs under its prefix, so it is left out rather than reported as deleting all of them. + /// + internal static IReadOnlyList ToDeletionRules(IEnumerable rules, + string container) + { + var result = new List(); + + foreach (var rule in rules ?? []) + { + var filters = rule.Definition?.Filters; + if (filters?.BlobIndexMatch?.Count > 0) continue; + if (filters?.BlobTypes?.Count > 0 && !filters.BlobTypes.Contains("blockBlob")) continue; + + var delete = rule.Definition?.Actions?.BaseBlob?.Delete; + var days = delete?.DaysAfterModificationGreaterThan ?? delete?.DaysAfterCreationGreaterThan; + if (days is not > 0) continue; + + // No prefix covers every container in the account; otherwise only this container's prefixes count. + var prefixes = filters?.PrefixMatch?.Count > 0 + ? filters.PrefixMatch + .Where(p => p == container || p.StartsWith(container + "/", StringComparison.Ordinal)) + .Select(p => p == container ? string.Empty : p[(container.Length + 1)..]) + .ToList() + : [string.Empty]; + + result.AddRange(prefixes.Select(prefix => new CloudFilesLifecycleRule + { + Id = rule.Name, + Prefix = prefix, + Days = (int)Math.Ceiling(days.Value), + // Absent means enabled. + Enabled = rule.IsEnabled ?? true + })); + } + + return result; + } +} diff --git a/SW.CloudFiles.AS/CloudFilesService.cs b/SW.CloudFiles.AS/CloudFilesService.cs index d8cadc5..a4abfb9 100644 --- a/SW.CloudFiles.AS/CloudFilesService.cs +++ b/SW.CloudFiles.AS/CloudFilesService.cs @@ -2,8 +2,12 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Threading; using System.Threading.Tasks; +using Azure; using Azure.Identity; +using Azure.ResourceManager.Storage; +using Azure.ResourceManager.Storage.Models; using Azure.Storage; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -13,11 +17,14 @@ namespace SW.CloudFiles.AS; -/// Azure Blob Storage implementation of . -public class CloudFilesService(AzureCloudFilesOptions cloudFilesOptions, BlobContainerClient blobContainerClient) : IDisposable, ICloudFilesService +/// Azure Blob Storage implementation of and . +public class CloudFilesService(AzureCloudFilesOptions cloudFilesOptions, BlobContainerClient blobContainerClient) + : IDisposable, ICloudFilesService, ICloudFilesLifecycle { private readonly AzureCloudFilesOptions cloudFilesOptions = cloudFilesOptions; private readonly BlobContainerClient blobContainerClient = blobContainerClient; + private readonly Lazy managementPolicy = + new(cloudFilesOptions.ManagementPolicy); /// public async Task WriteAsync(Stream inputStream, WriteFileSettings settings) @@ -181,6 +188,36 @@ public async Task DeleteAsync(string key) return true; } + /// + public async Task GetLifecycleAsync(CancellationToken cancellationToken = default) + { + var lifecycle = new CloudFilesLifecycle { Provider = "Azure", Bucket = cloudFilesOptions.BucketName }; + if (!cloudFilesOptions.CanManage()) + { + lifecycle.Unavailable = + "Azure deletes blobs through the storage account's lifecycle management policy, which can only be " + + "read or created once CloudFiles:SubscriptionId and CloudFiles:ResourceGroupName are set, with a " + + "role allowed to manage it (such as Storage Account Contributor). Until then the temp rules aren't added, " + + "and a policy set up by hand can't be read."; + return lifecycle; + } + + IEnumerable rules; + try + { + rules = (await managementPolicy.Value.GetAsync(cancellationToken)).Value.Data.Rules; + } + catch (RequestFailedException ex) when (ex.Status == 404 && ex.ErrorCode == "ManagementPolicyNotFound") + { + // An account that never had a policy has no rules. Any other 404 — a wrong account or + // resource group — is a real error and is left to surface. + rules = new List(); + } + + lifecycle.Rules = AzureLifecycleRules.ToDeletionRules(rules, cloudFilesOptions.BucketName); + return lifecycle; + } + /// public void Dispose() { diff --git a/SW.CloudFiles.AS/SW.CloudFiles.AS.csproj b/SW.CloudFiles.AS/SW.CloudFiles.AS.csproj index af73cca..6195a29 100644 --- a/SW.CloudFiles.AS/SW.CloudFiles.AS.csproj +++ b/SW.CloudFiles.AS/SW.CloudFiles.AS.csproj @@ -19,8 +19,11 @@ + + - + @@ -29,4 +32,9 @@ + + + + + diff --git a/SW.CloudFiles.GC.Extensions/IServiceCollectionExtensions.cs b/SW.CloudFiles.GC.Extensions/IServiceCollectionExtensions.cs index 2a03251..4f2b909 100644 --- a/SW.CloudFiles.GC.Extensions/IServiceCollectionExtensions.cs +++ b/SW.CloudFiles.GC.Extensions/IServiceCollectionExtensions.cs @@ -41,6 +41,7 @@ public static StorageClient BuildGoogleCloudStorageClient(this GoogleCloudFilesO /// Registers Google Cloud Storage as the implementation. /// On startup, ensures the bucket exists and — unless /// is true — creates delete lifecycle rules for the temp1/, temp7/, temp30/, and temp365/ prefixes. + /// Also registers , which reads the bucket's rules back. /// public static IServiceCollection AddGoogleCloudFiles(this IServiceCollection serviceCollection, Action configure = null) @@ -62,7 +63,9 @@ public static IServiceCollection AddGoogleCloudFiles(this IServiceCollection ser var urlSigner = cloudFilesOptions.BuildUrlSigner(); - serviceCollection.AddScoped(); + serviceCollection.AddScoped(); + serviceCollection.AddScoped(sp => sp.GetRequiredService()); + serviceCollection.AddScoped(sp => sp.GetRequiredService()); serviceCollection.AddSingleton(cloudFilesOptions); serviceCollection.AddSingleton(cloudFilesOptions); serviceCollection.AddSingleton(client); diff --git a/SW.CloudFiles.GC.UnitTests/GoogleLifecycleRulesTests.cs b/SW.CloudFiles.GC.UnitTests/GoogleLifecycleRulesTests.cs new file mode 100644 index 0000000..e623862 --- /dev/null +++ b/SW.CloudFiles.GC.UnitTests/GoogleLifecycleRulesTests.cs @@ -0,0 +1,45 @@ +using Google.Apis.Storage.v1.Data; +using SW.CloudFiles.GC; + +namespace SW.CloudFiles.GC.UnitTests; + +/// Pure rule logic, no bucket needed — unlike the other test in this project. +[TestClass] +public class GoogleLifecycleRulesTests +{ + private static Bucket.LifecycleData.RuleData Delete(int age, params string[] prefixes) => new() + { + Action = new Bucket.LifecycleData.RuleData.ActionData { Type = "Delete" }, + Condition = new Bucket.LifecycleData.RuleData.ConditionData + { + Age = age, + MatchesPrefix = prefixes.Length > 0 ? prefixes.ToList() : null + } + }; + + [TestMethod] + public void Reads_one_rule_per_matched_prefix() + { + var rules = GoogleLifecycleRules.ToDeletionRules([Delete(30, "temp30/", "scratch/")]); + + CollectionAssert.AreEquivalent(new[] { "temp30/", "scratch/" }, rules.Select(r => r.Prefix).ToArray()); + Assert.IsTrue(rules.All(r => r.Days == 30 && r.Enabled)); + } + + [TestMethod] + public void A_rule_with_no_prefix_covers_the_whole_bucket() + { + Assert.AreEqual(string.Empty, GoogleLifecycleRules.ToDeletionRules([Delete(7)]).Single().Prefix); + } + + [TestMethod] + public void Storage_class_changes_and_narrowed_rules_are_left_out() + { + var tiering = Delete(30, "cold/"); + tiering.Action.Type = "SetStorageClass"; + var suffixed = Delete(30, "temp30/"); + suffixed.Condition.MatchesSuffix = [".log"]; + + Assert.AreEqual(0, GoogleLifecycleRules.ToDeletionRules([tiering, suffixed]).Count); + } +} diff --git a/SW.CloudFiles.GC/CloudFilesService.cs b/SW.CloudFiles.GC/CloudFilesService.cs index 3988874..52f57e1 100644 --- a/SW.CloudFiles.GC/CloudFilesService.cs +++ b/SW.CloudFiles.GC/CloudFilesService.cs @@ -3,15 +3,16 @@ using System.Collections.ObjectModel; using System.IO; using System.Net; +using System.Threading; using System.Threading.Tasks; using Google.Cloud.Storage.V1; using SW.PrimitiveTypes; namespace SW.CloudFiles.GC; -/// Google Cloud Storage implementation of . +/// Google Cloud Storage implementation of and . public class CloudFilesService(GoogleCloudFilesOptions options, StorageClient storageClient, UrlSigner urlSigner) - : ICloudFilesService + : ICloudFilesService, ICloudFilesLifecycle { /// public async Task WriteAsync(Stream inputStream, WriteFileSettings settings) @@ -110,4 +111,17 @@ public async Task DeleteAsync(string key) return false; } } + + /// + public async Task GetLifecycleAsync(CancellationToken cancellationToken = default) + { + var bucket = await storageClient.GetBucketAsync(options.BucketName, cancellationToken: cancellationToken); + + return new CloudFilesLifecycle + { + Provider = "Google", + Bucket = options.BucketName, + Rules = GoogleLifecycleRules.ToDeletionRules(bucket.Lifecycle?.Rule) + }; + } } diff --git a/SW.CloudFiles.GC/GoogleLifecycleRules.cs b/SW.CloudFiles.GC/GoogleLifecycleRules.cs new file mode 100644 index 0000000..9f91fce --- /dev/null +++ b/SW.CloudFiles.GC/GoogleLifecycleRules.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.Linq; +using Google.Apis.Storage.v1.Data; +using SW.PrimitiveTypes; + +namespace SW.CloudFiles.GC; + +/// How a Cloud Storage bucket's lifecycle rules read as deletion rules. +internal static class GoogleLifecycleRules +{ + /// + /// The rules that delete objects by age. Any other condition (storage class, suffix, dates, versions) + /// narrows a rule to some of the objects under its prefixes, so such rules are left out. + /// + internal static IReadOnlyList ToDeletionRules(IEnumerable rules) + { + var result = new List(); + + foreach (var rule in rules ?? Enumerable.Empty()) + { + var condition = rule.Condition; + if (rule.Action?.Type != "Delete" || condition?.Age is not > 0) continue; + if (condition.CreatedBefore != null || condition.CustomTimeBefore != null || + condition.DaysSinceCustomTime != null || condition.DaysSinceNoncurrentTime != null || + condition.IsLive != null || condition.NoncurrentTimeBefore != null || + condition.NumNewerVersions != null || condition.MatchesStorageClass?.Count > 0 || + condition.MatchesSuffix?.Count > 0) + continue; + + var prefixes = condition.MatchesPrefix?.Count > 0 ? condition.MatchesPrefix : [string.Empty]; + result.AddRange(prefixes.Select(prefix => new CloudFilesLifecycleRule + { + // Cloud Storage rules have no names. + Id = null, + Prefix = prefix ?? string.Empty, + Days = condition.Age.Value, + // A rule is either present or absent; there is no switch to turn one off. + Enabled = true + })); + } + + return result; + } +} diff --git a/SW.CloudFiles.GC/SW.CloudFiles.GC.csproj b/SW.CloudFiles.GC/SW.CloudFiles.GC.csproj index 62da75a..450cdd1 100644 --- a/SW.CloudFiles.GC/SW.CloudFiles.GC.csproj +++ b/SW.CloudFiles.GC/SW.CloudFiles.GC.csproj @@ -19,7 +19,7 @@ - + @@ -27,4 +27,8 @@ + + + + diff --git a/SW.CloudFiles.LocalTests.Extensions/IServiceCollectionExtensions.cs b/SW.CloudFiles.LocalTests.Extensions/IServiceCollectionExtensions.cs index e817060..27a11d6 100644 --- a/SW.CloudFiles.LocalTests.Extensions/IServiceCollectionExtensions.cs +++ b/SW.CloudFiles.LocalTests.Extensions/IServiceCollectionExtensions.cs @@ -55,6 +55,7 @@ public static IServiceCollection AddLocalTestsCloudFiles(this IServiceCollection serviceCollection.AddSingleton(cloudFilesOptions); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(sp => sp.GetRequiredService()); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); return serviceCollection; } diff --git a/SW.CloudFiles.LocalTests/SW.CloudFiles.LocalTests.csproj b/SW.CloudFiles.LocalTests/SW.CloudFiles.LocalTests.csproj index a5984c5..f25f759 100644 --- a/SW.CloudFiles.LocalTests/SW.CloudFiles.LocalTests.csproj +++ b/SW.CloudFiles.LocalTests/SW.CloudFiles.LocalTests.csproj @@ -18,7 +18,7 @@ - + diff --git a/SW.CloudFiles.LocalTests/Services/CloudFilesService.cs b/SW.CloudFiles.LocalTests/Services/CloudFilesService.cs index 22f6135..737e61f 100644 --- a/SW.CloudFiles.LocalTests/Services/CloudFilesService.cs +++ b/SW.CloudFiles.LocalTests/Services/CloudFilesService.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using SW.PrimitiveTypes; @@ -20,7 +21,7 @@ namespace SW.CloudFiles.LocalTests; /// in your test teardown to remove all files written during the /// test run. /// -public class CloudFilesService(LocalTestsCloudFilesOptions options) : ICloudFilesService, IDisposable +public class CloudFilesService(LocalTestsCloudFilesOptions options) : ICloudFilesService, ICloudFilesLifecycle, IDisposable { private readonly string _root = options.ResolvedStoragePath; @@ -171,4 +172,11 @@ public void Cleanup() /// public void Dispose() { } + + /// + /// Nothing deletes files on the local filesystem by age: the temp-prefix rules the cloud providers + /// create don't exist here, which is what this reports. + /// + public Task GetLifecycleAsync(CancellationToken cancellationToken = default) => + Task.FromResult(new CloudFilesLifecycle { Provider = "Local", Bucket = options.BucketName }); } diff --git a/SW.CloudFiles.OC.Extensions/IServiceCollectionExtensions.cs b/SW.CloudFiles.OC.Extensions/IServiceCollectionExtensions.cs index ada0983..eb45701 100644 --- a/SW.CloudFiles.OC.Extensions/IServiceCollectionExtensions.cs +++ b/SW.CloudFiles.OC.Extensions/IServiceCollectionExtensions.cs @@ -21,7 +21,8 @@ public static class IServiceCollectionExtensions /// Registers Oracle Cloud Infrastructure Object Storage as the implementation. /// On startup, writes OCI credentials to temporary files and — unless /// is true — creates delete lifecycle rules - /// for the temp1/, temp7/, temp30/, and temp365/ prefixes on the bucket. + /// for the temp1/, temp7/, temp30/, and temp365/ prefixes on the bucket. Also registers + /// , which reads the bucket's rules back. /// public static IServiceCollection AddOracleCloudFiles(this IServiceCollection serviceCollection, Action configure = null) @@ -51,7 +52,9 @@ public static IServiceCollection AddOracleCloudFiles(this IServiceCollection ser if (!cloudFilesOptions.DisableAutoLifecycle) EnsureLifecycleRules(cloudFilesOptions); - serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); serviceCollection.AddSingleton(cloudFilesOptions); serviceCollection.AddSingleton(cloudFilesOptions); return serviceCollection; diff --git a/SW.CloudFiles.OC.UnitTest/OracleLifecycleRulesTests.cs b/SW.CloudFiles.OC.UnitTest/OracleLifecycleRulesTests.cs new file mode 100644 index 0000000..1b907be --- /dev/null +++ b/SW.CloudFiles.OC.UnitTest/OracleLifecycleRulesTests.cs @@ -0,0 +1,70 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Oci.ObjectstorageService.Models; +using SW.CloudFiles.OC; + +namespace SW.CloudFiles.OC.UnitTest +{ + /// Pure rule logic, no bucket needed — unlike the other tests in this project. + [TestClass] + public class OracleLifecycleRulesTests + { + private static ObjectLifecycleRule Delete(string name, long amount, ObjectLifecycleRule.TimeUnitEnum unit, + params string[] prefixes) => new ObjectLifecycleRule + { + Name = name, + Action = "DELETE", + TimeAmount = amount, + TimeUnit = unit, + IsEnabled = true, + Target = "objects", + ObjectNameFilter = new ObjectNameFilter { InclusionPrefixes = prefixes.ToList() } + }; + + [TestMethod] + public void Reads_one_rule_per_included_prefix() + { + var rules = OracleLifecycleRules.ToDeletionRules(new[] + { + Delete("delete-temp", 30, ObjectLifecycleRule.TimeUnitEnum.Days, "temp30/", "scratch/") + }); + + CollectionAssert.AreEquivalent(new[] { "temp30/", "scratch/" }, rules.Select(r => r.Prefix).ToArray()); + Assert.IsTrue(rules.All(r => r.Days == 30 && r.Enabled)); + } + + [TestMethod] + public void Years_count_as_365_days() + { + var rule = OracleLifecycleRules.ToDeletionRules(new[] + { + Delete("delete-temp365", 1, ObjectLifecycleRule.TimeUnitEnum.Years, "temp365/") + }).Single(); + + Assert.AreEqual(365, rule.Days); + } + + [TestMethod] + public void A_rule_with_no_prefixes_covers_the_whole_bucket() + { + var rule = OracleLifecycleRules.ToDeletionRules(new[] + { + Delete("delete-all", 7, ObjectLifecycleRule.TimeUnitEnum.Days) + }).Single(); + + Assert.AreEqual(string.Empty, rule.Prefix); + } + + [TestMethod] + public void Archive_actions_and_pattern_filtered_rules_are_left_out() + { + var archive = Delete("archive", 30, ObjectLifecycleRule.TimeUnitEnum.Days, "cold/"); + archive.Action = "ARCHIVE"; + var patterned = Delete("patterned", 30, ObjectLifecycleRule.TimeUnitEnum.Days, "temp30/"); + patterned.ObjectNameFilter.InclusionPatterns = new List { "*.log" }; + + Assert.AreEqual(0, OracleLifecycleRules.ToDeletionRules(new[] { archive, patterned }).Count); + } + } +} diff --git a/SW.CloudFiles.OC/CloudFilesService.cs b/SW.CloudFiles.OC/CloudFilesService.cs index 923cab5..bdab1a0 100644 --- a/SW.CloudFiles.OC/CloudFilesService.cs +++ b/SW.CloudFiles.OC/CloudFilesService.cs @@ -2,10 +2,13 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net; using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Oci.Common.Auth; +using Oci.Common.Model; using Oci.ObjectstorageService; using Oci.ObjectstorageService.Models; using Oci.ObjectstorageService.Requests; @@ -14,8 +17,8 @@ namespace SW.CloudFiles.OC { - /// Oracle Cloud Infrastructure Object Storage implementation of . - public class CloudFilesService : ICloudFilesService, IDisposable + /// Oracle Cloud Infrastructure Object Storage implementation of and . + public class CloudFilesService : ICloudFilesService, ICloudFilesLifecycle, IDisposable { private readonly OracleCloudFilesOptions _cloudFilesOptions; private readonly UploadManager _uploadManager; @@ -184,6 +187,33 @@ await _client.DeleteObject(new DeleteObjectRequest } } + /// + public async Task GetLifecycleAsync(CancellationToken cancellationToken = default) + { + IEnumerable rules; + try + { + var response = await _client.GetObjectLifecyclePolicy(new GetObjectLifecyclePolicyRequest + { + BucketName = _cloudFilesOptions.BucketName, + NamespaceName = _cloudFilesOptions.NamespaceName + }, cancellationToken: cancellationToken); + rules = response.ObjectLifecyclePolicy?.Items ?? new List(); + } + catch (OciException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // A bucket that never had a policy has none to return. + rules = new List(); + } + + return new CloudFilesLifecycle + { + Provider = "Oracle", + Bucket = _cloudFilesOptions.BucketName, + Rules = OracleLifecycleRules.ToDeletionRules(rules) + }; + } + /// public void Dispose() => _client?.Dispose(); } diff --git a/SW.CloudFiles.OC/OracleLifecycleRules.cs b/SW.CloudFiles.OC/OracleLifecycleRules.cs new file mode 100644 index 0000000..c1931e6 --- /dev/null +++ b/SW.CloudFiles.OC/OracleLifecycleRules.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.Linq; +using Oci.ObjectstorageService.Models; +using SW.PrimitiveTypes; + +namespace SW.CloudFiles.OC +{ + /// How an Object Storage lifecycle policy reads as deletion rules. + internal static class OracleLifecycleRules + { + /// + /// The rules that delete objects by age. Archive and tiering actions don't delete anything, and a + /// rule narrowed by name patterns only reaches some of the objects under its prefixes, so both are + /// left out. + /// + internal static IReadOnlyList ToDeletionRules(IEnumerable rules) + { + var result = new List(); + + foreach (var rule in rules ?? Enumerable.Empty()) + { + if (rule.Action != "DELETE" || rule.TimeAmount is not > 0) continue; + if (rule.Target != null && rule.Target != "objects") continue; + + var filter = rule.ObjectNameFilter; + if (filter?.InclusionPatterns?.Count > 0 || filter?.ExclusionPatterns?.Count > 0) continue; + + var days = (int)rule.TimeAmount.Value * + (rule.TimeUnit == ObjectLifecycleRule.TimeUnitEnum.Years ? 365 : 1); + var prefixes = filter?.InclusionPrefixes?.Count > 0 + ? filter.InclusionPrefixes + : new List { string.Empty }; + + result.AddRange(prefixes.Select(prefix => new CloudFilesLifecycleRule + { + Id = rule.Name, + Prefix = prefix ?? string.Empty, + Days = days, + Enabled = rule.IsEnabled ?? false + })); + } + + return result; + } + } +} diff --git a/SW.CloudFiles.OC/SW.CloudFiles.OC.csproj b/SW.CloudFiles.OC/SW.CloudFiles.OC.csproj index 974efbc..d0216df 100644 --- a/SW.CloudFiles.OC/SW.CloudFiles.OC.csproj +++ b/SW.CloudFiles.OC/SW.CloudFiles.OC.csproj @@ -20,7 +20,7 @@ - + @@ -28,4 +28,8 @@ + + + + diff --git a/SW.CloudFiles.S3.Extensions/IServiceCollectionExtensions.cs b/SW.CloudFiles.S3.Extensions/IServiceCollectionExtensions.cs index d4b210c..1ec1eab 100644 --- a/SW.CloudFiles.S3.Extensions/IServiceCollectionExtensions.cs +++ b/SW.CloudFiles.S3.Extensions/IServiceCollectionExtensions.cs @@ -6,8 +6,6 @@ using Nito.AsyncEx.Synchronous; using SW.PrimitiveTypes; using System; -using System.Collections.Generic; -using System.Linq; using SW.CloudFiles.S3; namespace SW.CloudFiles.Extensions; @@ -19,7 +17,9 @@ public static class IServiceCollectionExtensions /// Registers an S3-compatible provider as the implementation. /// On startup, creates the bucket if it does not exist and — unless /// is true — ensures delete lifecycle rules - /// exist for the temp1/, temp7/, temp30/, and temp365/ prefixes (1, 7, 30, and 365 days respectively). + /// exist for the temp1/, temp7/, temp30/, and temp365/ prefixes (1, 7, 30, and 365 days respectively), + /// keeping every other rule already on the bucket. Also registers , + /// which reads the bucket's rules back. /// public static IServiceCollection AddS3CloudFiles(this IServiceCollection serviceCollection, Action configure = null) @@ -52,85 +52,24 @@ public static IServiceCollection AddS3CloudFiles(this IServiceCollection service BucketName = cloudFilesOptions.BucketName }).WaitAndUnwrapException().Configuration; - var newRules = new List { }; - - if (config.Rules?.FirstOrDefault(r => r.Id == "temp1") == null || - config.Rules?.FirstOrDefault(r => r.Id == "temp1")?.Status != LifecycleRuleStatus.Enabled) - newRules.Add(new LifecycleRule - { - Id = "temp1", - Expiration = new LifecycleRuleExpiration { Days = 1 }, - Filter = new LifecycleFilter() - { - LifecycleFilterPredicate = new LifecyclePrefixPredicate() - { - Prefix = "temp1/" - } - }, - Status = LifecycleRuleStatus.Enabled - }); - - if (config.Rules?.FirstOrDefault(r => r.Id == "temp7") == null || - config.Rules?.FirstOrDefault(r => r.Id == "temp7")?.Status != LifecycleRuleStatus.Enabled) - newRules.Add(new LifecycleRule - { - Id = "temp7", - Expiration = new LifecycleRuleExpiration { Days = 7 }, - Filter = new LifecycleFilter() - { - LifecycleFilterPredicate = new LifecyclePrefixPredicate() - { - Prefix = "temp7/" - } - }, - Status = LifecycleRuleStatus.Enabled - }); - - if (config.Rules?.FirstOrDefault(r => r.Id == "temp30") == null || - config.Rules?.FirstOrDefault(r => r.Id == "temp30")?.Status != LifecycleRuleStatus.Enabled) - newRules.Add(new LifecycleRule - { - Id = "temp30", - Expiration = new LifecycleRuleExpiration { Days = 30 }, - Filter = new LifecycleFilter - { - LifecycleFilterPredicate = new LifecyclePrefixPredicate - { - Prefix = "temp30/" - } - }, - Status = LifecycleRuleStatus.Enabled - }); - - if (config.Rules?.FirstOrDefault(r => r.Id == "temp365") == null || - config.Rules?.FirstOrDefault(r => r.Id == "temp365")?.Status != LifecycleRuleStatus.Enabled) - newRules.Add(new LifecycleRule - { - Id = "temp365", - Expiration = new LifecycleRuleExpiration { Days = 365 }, - Filter = new LifecycleFilter() - { - LifecycleFilterPredicate = new LifecyclePrefixPredicate() - { - Prefix = "temp365/" - } - }, - Status = LifecycleRuleStatus.Enabled - }); - - if (newRules.Count > 0) + // The whole rule set goes back, not just the missing temp rules: S3 replaces the entire + // configuration on every write, so sending only the additions deleted every other rule. + var rules = S3LifecycleRules.WithTempRules(config?.Rules); + if (rules != null) { client.PutLifecycleConfigurationAsync(new PutLifecycleConfigurationRequest { BucketName = cloudFilesOptions.BucketName, - Configuration = new LifecycleConfiguration { Rules = newRules } + Configuration = new LifecycleConfiguration { Rules = rules } }).WaitAndUnwrapException(); } } } serviceCollection.AddSingleton(cloudFilesOptions); - serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); return serviceCollection; } diff --git a/SW.CloudFiles.S3/S3LifecycleRules.cs b/SW.CloudFiles.S3/S3LifecycleRules.cs new file mode 100644 index 0000000..98a1912 --- /dev/null +++ b/SW.CloudFiles.S3/S3LifecycleRules.cs @@ -0,0 +1,91 @@ +using System.Collections.Generic; +using System.Linq; +using Amazon.S3; +using Amazon.S3.Model; +using SW.PrimitiveTypes; + +namespace SW.CloudFiles.S3; + +/// The temp-prefix rules this library keeps on a bucket, and how S3 rules read as deletion rules. +internal static class S3LifecycleRules +{ + /// Files under these prefixes are deleted after the given number of days. + internal static readonly (string Id, string Prefix, int Days)[] Temp = + [ + ("temp1", "temp1/", 1), + ("temp7", "temp7/", 7), + ("temp30", "temp30/", 30), + ("temp365", "temp365/", 365) + ]; + + /// + /// The bucket's whole rule set with every missing or disabled temp rule put back, or null when + /// nothing needs changing. S3 replaces the entire configuration on every write, so the rules this + /// library doesn't own have to be sent back too or they are deleted. + /// + internal static List WithTempRules(IEnumerable existing) + { + var rules = existing?.ToList() ?? []; + var changed = false; + + foreach (var (id, prefix, days) in Temp) + { + if (rules.Any(r => r.Id == id && r.Status == LifecycleRuleStatus.Enabled)) continue; + + rules.RemoveAll(r => r.Id == id); + rules.Add(new LifecycleRule + { + Id = id, + Expiration = new LifecycleRuleExpiration { Days = days }, + Filter = new LifecycleFilter + { + LifecycleFilterPredicate = new LifecyclePrefixPredicate { Prefix = prefix } + }, + Status = LifecycleRuleStatus.Enabled + }); + changed = true; + } + + return changed ? rules : null; + } + + /// + /// The rules that delete files by age. A rule filtered on tags or object size only reaches some of + /// the files under its prefix, so it is left out rather than reported as deleting all of them. + /// + internal static IReadOnlyList ToDeletionRules(IEnumerable rules) + { + var result = new List(); + + foreach (var rule in rules ?? []) + { + if (rule.Expiration == null || rule.Expiration.Days <= 0) continue; + + string prefix; + switch (rule.Filter?.LifecycleFilterPredicate) + { + case LifecyclePrefixPredicate predicate: + prefix = predicate.Prefix ?? string.Empty; + break; + case null: + // No filter predicate: the older rule-level prefix, or the whole bucket. +#pragma warning disable CS0618 + prefix = rule.Prefix ?? string.Empty; +#pragma warning restore CS0618 + break; + default: + continue; + } + + result.Add(new CloudFilesLifecycleRule + { + Id = rule.Id, + Prefix = prefix, + Days = rule.Expiration.Days, + Enabled = rule.Status == LifecycleRuleStatus.Enabled + }); + } + + return result; + } +} diff --git a/SW.CloudFiles.S3/SW.CloudFiles.S3.csproj b/SW.CloudFiles.S3/SW.CloudFiles.S3.csproj index c57640a..c724a50 100644 --- a/SW.CloudFiles.S3/SW.CloudFiles.S3.csproj +++ b/SW.CloudFiles.S3/SW.CloudFiles.S3.csproj @@ -32,9 +32,14 @@ 10.0.0.0 + + + + + - + diff --git a/SW.CloudFiles.S3/Services/CloudFilesService.cs b/SW.CloudFiles.S3/Services/CloudFilesService.cs index ee80076..3e68ea1 100644 --- a/SW.CloudFiles.S3/Services/CloudFilesService.cs +++ b/SW.CloudFiles.S3/Services/CloudFilesService.cs @@ -4,13 +4,15 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; +using System.Threading; using System.Threading.Tasks; namespace SW.CloudFiles.S3; -/// S3-compatible implementation of . -public class CloudFilesService : IDisposable, ICloudFilesService +/// S3-compatible implementation of and . +public class CloudFilesService : IDisposable, ICloudFilesService, ICloudFilesLifecycle { private const string metadataPrefix = "x-amz-meta-"; private readonly CloudFilesOptions cloudFilesOptions; @@ -236,6 +238,31 @@ public async Task> ListAsync(string prefix) } + /// + public async Task GetLifecycleAsync(CancellationToken cancellationToken = default) + { + IEnumerable rules; + try + { + var response = await client.GetLifecycleConfigurationAsync(new GetLifecycleConfigurationRequest + { + BucketName = cloudFilesOptions.BucketName + }, cancellationToken); + rules = response.Configuration?.Rules ?? Enumerable.Empty(); + } + catch (AmazonS3Exception ex) when (ex.ErrorCode == "NoSuchLifecycleConfiguration") + { + rules = Enumerable.Empty(); + } + + return new CloudFilesLifecycle + { + Provider = "S3", + Bucket = cloudFilesOptions.BucketName, + Rules = S3LifecycleRules.ToDeletionRules(rules) + }; + } + /// public void Dispose() { diff --git a/SW.CloudFiles.UnitTests/S3LifecycleRulesTests.cs b/SW.CloudFiles.UnitTests/S3LifecycleRulesTests.cs new file mode 100644 index 0000000..b0c3a71 --- /dev/null +++ b/SW.CloudFiles.UnitTests/S3LifecycleRulesTests.cs @@ -0,0 +1,103 @@ +using System.Collections.Generic; +using System.Linq; +using Amazon.S3; +using Amazon.S3.Model; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.CloudFiles.S3; + +namespace SW.CloudFiles.UnitTests; + +/// Pure rule logic, no bucket needed — unlike the other tests in this project. +[TestClass] +public class S3LifecycleRulesTests +{ + private static LifecycleRule PrefixRule(string id, string prefix, int days, bool enabled = true) => new() + { + Id = id, + Expiration = new LifecycleRuleExpiration { Days = days }, + Filter = new LifecycleFilter { LifecycleFilterPredicate = new LifecyclePrefixPredicate { Prefix = prefix } }, + Status = enabled ? LifecycleRuleStatus.Enabled : LifecycleRuleStatus.Disabled + }; + + [TestMethod] + public void Adding_the_temp_rules_keeps_every_rule_already_on_the_bucket() + { + var existing = new List { PrefixRule("logs", "logs/", 14), PrefixRule("temp1", "temp1/", 1) }; + + var rules = S3LifecycleRules.WithTempRules(existing); + + CollectionAssert.AreEquivalent( + new[] { "logs", "temp1", "temp7", "temp30", "temp365" }, + rules.Select(r => r.Id).ToArray()); + Assert.AreEqual(14, rules.Single(r => r.Id == "logs").Expiration.Days); + } + + [TestMethod] + public void Nothing_to_write_when_every_temp_rule_is_there_and_enabled() + { + var existing = S3LifecycleRules.Temp.Select(t => PrefixRule(t.Id, t.Prefix, t.Days)).ToList(); + + Assert.IsNull(S3LifecycleRules.WithTempRules(existing)); + } + + [TestMethod] + public void A_disabled_temp_rule_is_replaced_not_duplicated() + { + var existing = new List { PrefixRule("temp30", "temp30/", 30, enabled: false) }; + + var rules = S3LifecycleRules.WithTempRules(existing); + + var temp30 = rules.Where(r => r.Id == "temp30").ToList(); + Assert.AreEqual(1, temp30.Count); + Assert.AreEqual(LifecycleRuleStatus.Enabled, temp30[0].Status); + } + + [TestMethod] + public void Reads_prefix_filtered_expirations_as_deletion_rules() + { + var rules = S3LifecycleRules.ToDeletionRules([PrefixRule("temp30", "temp30/", 30)]); + + var rule = rules.Single(); + Assert.AreEqual("temp30/", rule.Prefix); + Assert.AreEqual(30, rule.Days); + Assert.IsTrue(rule.Enabled); + } + + [TestMethod] + public void A_rule_without_a_filter_covers_the_whole_bucket() + { + var rules = S3LifecycleRules.ToDeletionRules([new LifecycleRule + { + Id = "everything", + Expiration = new LifecycleRuleExpiration { Days = 90 }, + Status = LifecycleRuleStatus.Enabled + }]); + + Assert.AreEqual(string.Empty, rules.Single().Prefix); + } + + [TestMethod] + public void Tag_filtered_and_non_expiring_rules_are_left_out() + { + var rules = S3LifecycleRules.ToDeletionRules([ + new LifecycleRule + { + Id = "tagged", + Expiration = new LifecycleRuleExpiration { Days = 7 }, + Filter = new LifecycleFilter + { + LifecycleFilterPredicate = new LifecycleTagPredicate { Tag = new Tag { Key = "k", Value = "v" } } + }, + Status = LifecycleRuleStatus.Enabled + }, + new LifecycleRule + { + Id = "transition-only", + Filter = new LifecycleFilter { LifecycleFilterPredicate = new LifecyclePrefixPredicate { Prefix = "cold/" } }, + Status = LifecycleRuleStatus.Enabled + } + ]); + + Assert.AreEqual(0, rules.Count); + } +}