diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs index dbbe3e22d9..725bd29064 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs @@ -98,6 +98,80 @@ public static string BundleRegistryKey(string productGroup) => public static string ChangelogRegistryKey(string poolGroup) => $"{ChangelogPrefix}{poolGroup}/{RegistryFileName}"; + /// + /// The notes-index key for one target within a repo: changelog/{org}/{repo}/notes-{target}.json. + /// Repo-level and branch-agnostic — all notes for a target, regardless of which branch they were authored on. + /// + public static string NotesIndexKey(string org, string repo, string target) => + $"{ChangelogPrefix}{org}/{repo}/notes-{target}.json"; + + /// + /// The S3 prefix that covers all branches and notes indexes of one repo: changelog/{org}/{repo}/. + /// Used by the notes reconciler to list the full repo tree. + /// + public static string RepoPrefix(string org, string repo) => + $"{ChangelogPrefix}{org}/{repo}/"; + + /// + /// Returns true when is a notes-index key of the form + /// changelog/{org}/{repo}/notes-{target}.json (exactly two group segments, then + /// a notes--prefixed JSON file with a non-empty target slug). + /// + public static bool IsNotesIndex(string key) + { + if (!key.StartsWith(ChangelogPrefix, StringComparison.Ordinal)) + return false; + if (!key.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + return false; + + var rest = key.AsSpan(ChangelogPrefix.Length); + + // Expect exactly {org}/{repo}/notes-{target}.json + var firstSlash = rest.IndexOf('/'); + if (firstSlash <= 0) + return false; + var org = rest[..firstSlash]; + + var afterOrg = rest[(firstSlash + 1)..]; + var secondSlash = afterOrg.IndexOf('/'); + if (secondSlash <= 0) + return false; + var repo = afterOrg[..secondSlash]; + + var file = afterOrg[(secondSlash + 1)..]; + + // No further slashes — this must be a direct child of {org}/{repo}/ + if (file.IndexOf('/') >= 0) + return false; + + // Validate org/repo segments and require the notes- prefix with a non-empty target slug + if (!IsValidSegment(org, SegmentKind.Org) || !IsValidSegment(repo, SegmentKind.RepoOrBranch)) + return false; + + const string notesPrefix = "notes-"; + if (!file.StartsWith(notesPrefix, StringComparison.Ordinal)) + return false; + + var targetSlug = file[notesPrefix.Length..^".json".Length]; + return targetSlug.Length > 0 && IsValidSegment(targetSlug, SegmentKind.RepoOrBranch); + } + + /// + /// Extracts the {org}/{repo} group from a changelog/{org}/{repo}/notes-{target}.json key. + /// Returns null when the key is not a valid notes-index key. + /// + public static string? ExtractNotesRepo(string key) + { + if (!IsNotesIndex(key)) + return null; + + var rest = key.AsSpan(ChangelogPrefix.Length); + var firstSlash = rest.IndexOf('/'); + var afterOrg = rest[(firstSlash + 1)..]; + var secondSlash = afterOrg.IndexOf('/'); + return $"{rest[..firstSlash]}/{afterOrg[..secondSlash]}"; + } + /// /// Extracts the product group from a bundle/{product}/{file} key, or null when /// is not a bundle key with a valid product segment ahead of the file name. diff --git a/src/infra/docs-lambda-changelog-scrubber/Program.cs b/src/infra/docs-lambda-changelog-scrubber/Program.cs index 6edd17f89a..ff7d7c3c35 100644 --- a/src/infra/docs-lambda-changelog-scrubber/Program.cs +++ b/src/infra/docs-lambda-changelog-scrubber/Program.cs @@ -56,7 +56,8 @@ async Task Handler(SQSEvent ev, ILambdaContext context) var scrubber = new ChangelogContentScrubber(logFactory, allowRepos); var reconciler = new BundleRegistryReconciler(logFactory, s3Client, publicBucketName, metrics: metrics); var shallowReconciler = new ShallowRegistryReconciler(logFactory, s3Client, publicBucketName, metrics: metrics); - var processor = new ScrubberProcessor(logFactory, s3Client, publicBucketName, scrubber, reconciler, shallowReconciler, metrics); + var notesReconciler = new NotesIndexReconciler(logFactory, s3Client, publicBucketName, metrics: metrics); + var processor = new ScrubberProcessor(logFactory, s3Client, publicBucketName, scrubber, reconciler, shallowReconciler, notesReconciler, metrics); var messages = ev.Records.Select(r => new ScrubberQueueMessage(r.MessageId, r.Body)).ToList(); var failedIds = await processor.ProcessAsync(messages, CancellationToken.None); diff --git a/src/services/Elastic.Changelog/Reconciliation/ChangelogScope.cs b/src/services/Elastic.Changelog/Reconciliation/ChangelogScope.cs index c81da6fba0..f42907ed27 100644 --- a/src/services/Elastic.Changelog/Reconciliation/ChangelogScope.cs +++ b/src/services/Elastic.Changelog/Reconciliation/ChangelogScope.cs @@ -8,7 +8,7 @@ namespace Elastic.Changelog.Reconciliation; -/// The two registry scope families in the changelog bucket key layout. +/// The registry scope families in the changelog bucket key layout. [JsonConverter(typeof(JsonStringEnumConverter))] public enum ChangelogScopeKind { @@ -16,15 +16,19 @@ public enum ChangelogScopeKind Bundle, /// An authoring-pool scope: changelog/{org}/{repo}/{branch}/…. - Changelog + Changelog, + + /// A repo-level notes scope: changelog/{org}/{repo}/ (branch-agnostic). + Notes } /// /// Identifies one registry scope in the changelog bundles bucket — a product bundle pool -/// (bundle/{product}/) or an authoring changelog pool -/// (changelog/{org}/{repo}/{branch}/) — and derives the scope's key prefix and -/// registry.json key. Segments are validated on construction via -/// , so a scope instance can always be composed into safe S3 keys. +/// (bundle/{product}/), an authoring changelog pool +/// (changelog/{org}/{repo}/{branch}/), or a repo-level notes scope +/// (changelog/{org}/{repo}/) — and derives the scope's key prefix. Segments are +/// validated on construction via , so a scope instance can +/// always be composed into safe S3 keys. /// public sealed record ChangelogScope { @@ -39,16 +43,20 @@ private ChangelogScope(ChangelogScopeKind kind, string group) /// /// The grouping segment(s): the product for a bundle scope, the - /// {org}/{repo}/{branch} prefix for a changelog scope. + /// {org}/{repo}/{branch} prefix for a changelog scope, or + /// {org}/{repo} for a notes scope. /// public string Group { get; } /// The S3 key prefix of every object in this scope, ending in /. - public string Prefix => Kind == ChangelogScopeKind.Bundle - ? $"{ChangelogKeys.BundlePrefix}{Group}/" - : $"{ChangelogKeys.ChangelogPrefix}{Group}/"; + public string Prefix => Kind switch + { + ChangelogScopeKind.Bundle => $"{ChangelogKeys.BundlePrefix}{Group}/", + ChangelogScopeKind.Notes => $"{ChangelogKeys.ChangelogPrefix}{Group}/", + _ => $"{ChangelogKeys.ChangelogPrefix}{Group}/" + }; - /// The S3 key of this scope's registry.json manifest. + /// The S3 key of this scope's registry.json manifest (bundle and changelog scopes only). public string RegistryKey => Kind == ChangelogScopeKind.Bundle ? ChangelogKeys.BundleRegistryKey(Group) : ChangelogKeys.ChangelogRegistryKey(Group); @@ -71,17 +79,27 @@ public static bool TryCreateChangelog(string? org, string? repo, string? branch, return scope is not null; } + /// Creates a notes scope for /; false when any segment is invalid. + public static bool TryCreateNotes(string? org, string? repo, [NotNullWhen(true)] out ChangelogScope? scope) + { + scope = ChangelogKeys.IsValidOrg(org) && ChangelogKeys.IsValidRepo(repo) + ? new ChangelogScope(ChangelogScopeKind.Notes, $"{org}/{repo}") + : null; + return scope is not null; + } + /// - /// Derives the scope an object key belongs to — bundle/{product}/{file} or - /// changelog/{org}/{repo}/{branch}/{file}, including the scope's own - /// registry.json key. False when the key sits outside both layouts or a segment - /// fails validation. + /// Derives the scope an object key belongs to — bundle/{product}/{file}, + /// changelog/{org}/{repo}/{branch}/{file}, or changelog/{org}/{repo}/notes-*.json. + /// False when the key sits outside all layouts or a segment fails validation. /// public static bool TryFromKey(string key, [NotNullWhen(true)] out ChangelogScope? scope) { scope = null; if (ChangelogKeys.ExtractBundleGroup(key) is { } product) scope = new ChangelogScope(ChangelogScopeKind.Bundle, product); + else if (ChangelogKeys.ExtractNotesRepo(key) is { } repo) + scope = new ChangelogScope(ChangelogScopeKind.Notes, repo); else if (ChangelogKeys.ExtractChangelogGroup(key) is { } pool) scope = new ChangelogScope(ChangelogScopeKind.Changelog, pool); return scope is not null; diff --git a/src/services/Elastic.Changelog/Reconciliation/NotesIndex.cs b/src/services/Elastic.Changelog/Reconciliation/NotesIndex.cs new file mode 100644 index 0000000000..b541de8a20 --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/NotesIndex.cs @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Elastic.Changelog.Reconciliation; + +/// +/// Notes index published at changelog/{org}/{repo}/notes-{target}.json. +/// Lists pool-relative paths of all note-*.yml fragments for one target, +/// across every branch of the repo. +/// +/// +/// Contents are paths, not bodies — the note files remain the single source of truth. +/// A stale index can only omit or over-list, never serve stale prose. Bundling a target +/// is therefore 1 GET for the index + one GET per listed note. +/// +public sealed record NotesIndex +{ + /// Pool-relative paths of notes for this target, e.g. ["main/note-slow-rollover.yml"]. + public required IReadOnlyList Notes { get; init; } +} + +[JsonSourceGenerationOptions( + WriteIndented = true, + PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull +)] +[JsonSerializable(typeof(NotesIndex))] +public sealed partial class NotesIndexJsonContext : JsonSerializerContext; diff --git a/src/services/Elastic.Changelog/Reconciliation/NotesIndexReconciler.cs b/src/services/Elastic.Changelog/Reconciliation/NotesIndexReconciler.cs new file mode 100644 index 0000000000..ad533dd846 --- /dev/null +++ b/src/services/Elastic.Changelog/Reconciliation/NotesIndexReconciler.cs @@ -0,0 +1,282 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using System.Text.Json; +using Amazon.S3; +using Amazon.S3.Model; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Microsoft.Extensions.Logging; + +namespace Elastic.Changelog.Reconciliation; + +/// +/// Rebuilds the per-target notes-{target}.json indexes for one repository by listing +/// all note-*.yml objects under changelog/{org}/{repo}/, reading each to extract +/// its target: values, and writing the affected indexes atomically. +/// +/// +/// A note may list products at multiple targets, so one note can appear in several indexes. +/// The index stores pool-relative paths ({branch}/note-{name}.yml) so the same file +/// name on two branches yields two distinct entries in the same index. +/// +public sealed class NotesIndexReconciler( + ILoggerFactory logFactory, + IAmazonS3 s3Client, + string publicBucketName, + string? sourceBucketName = null, + TimeSpan? retryBaseDelay = null, + ReconcileMetrics? metrics = null +) +{ + private const int MaxWriteAttempts = 5; + private const int MaxParallelReads = 8; + + private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly TimeSpan _retryBaseDelay = retryBaseDelay ?? TimeSpan.FromMilliseconds(200); + private readonly ReconcileMetrics _metrics = metrics ?? new ReconcileMetrics(); + private readonly string _sourceBucketName = sourceBucketName ?? publicBucketName; + + /// + /// Rebuilds all notes-{target}.json indexes for the given repository scope. + /// All currently published note-*.yml files across every branch are listed and + /// read to derive the target grouping; every affected index is then (re)written. + /// + public async Task ReconcileRepoAsync(ChangelogScope notesScope, Cancel ctx) + { + if (notesScope.Kind != ChangelogScopeKind.Notes) + throw new ArgumentException($"Notes reconcile requires a Notes scope; got '{notesScope}'.", nameof(notesScope)); + + _logger.LogInformation("Reconciling notes indexes for repo {Repo}", notesScope.Group); + + // List every note-*.yml under changelog/{org}/{repo}/ (all branches). + var noteObjects = await ListNoteFiles(notesScope, ctx); + _logger.LogDebug("Found {Count} note file(s) for {Repo}", noteObjects.Count, notesScope.Group); + + // Read each note to extract its targets. + // Using List per target; duplicates are removed at write time via Distinct(). + var byTarget = new Dictionary>(StringComparer.Ordinal); + foreach (var obj in noteObjects) + { + ctx.ThrowIfCancellationRequested(); + var poolRelativePath = obj.Key[notesScope.Prefix.Length..]; + var targets = await ExtractTargetsAsync(obj.Key, ctx); + foreach (var target in targets) + { + if (!byTarget.TryGetValue(target, out var paths)) + byTarget[target] = paths = []; + paths.Add(poolRelativePath); + } + } + + var groupParts = notesScope.Group.Split('/'); + var (org, repo) = (groupParts[0], groupParts[1]); + + // List existing notes-*.json indexes so we can remove obsolete ones. + var existingIndexKeys = await ListExistingNotesIndexes(notesScope, ctx); + + if (byTarget.Count == 0) + { + _logger.LogDebug("No targets found for repo {Repo}; removing any stale indexes", notesScope.Group); + await DeleteStaleIndexes(existingIndexKeys, [], org, repo, ctx); + return; + } + + // Write one index per target. DeleteStaleIndexes runs even if some writes fail — stale + // deletion is safe because we only remove targets absent from byTarget.Keys, which is + // independent of whether the new writes succeeded. + try + { + await Parallel.ForEachAsync( + byTarget, + new ParallelOptions { MaxDegreeOfParallelism = MaxParallelReads, CancellationToken = ctx }, + async (kvp, ct) => + { + var (target, paths) = kvp; + var indexKey = ChangelogKeys.NotesIndexKey(org, repo, target); + await WriteIndexAsync(indexKey, [.. paths.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal)], ct); + }); + } + finally + { + // Remove indexes whose targets are no longer present. + await DeleteStaleIndexes(existingIndexKeys, byTarget.Keys.ToHashSet(StringComparer.Ordinal), org, repo, ctx); + } + } + + private async Task> ListExistingNotesIndexes(ChangelogScope notesScope, Cancel ctx) + { + var request = new ListObjectsV2Request + { + BucketName = publicBucketName, + Prefix = notesScope.Prefix + }; + + var keys = new List(); + ListObjectsV2Response response; + do + { + response = await s3Client.ListObjectsV2Async(request, ctx); + foreach (var obj in response.S3Objects ?? []) + { + if (ChangelogKeys.IsNotesIndex(obj.Key)) + keys.Add(obj.Key); + } + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated == true); + + return keys; + } + + private async Task DeleteStaleIndexes( + IReadOnlyList existingKeys, + HashSet currentTargets, + string org, + string repo, + Cancel ctx) + { + // "changelog/{org}/{repo}/notes-" — the stable prefix shared by all notes-*.json keys for this repo. + var notesKeyPrefix = $"{ChangelogKeys.ChangelogPrefix}{org}/{repo}/notes-"; + + foreach (var key in existingKeys) + { + // Extract the target slug from the key to check if it's still needed. + if (!key.StartsWith(notesKeyPrefix, StringComparison.Ordinal)) + continue; + var targetSlug = key[notesKeyPrefix.Length..^".json".Length]; + if (currentTargets.Contains(targetSlug)) + continue; + + try + { + _ = await s3Client.DeleteObjectAsync(new DeleteObjectRequest + { + BucketName = publicBucketName, + Key = key + }, ctx); + _logger.LogInformation("Removed stale notes index {Key}", key); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to delete stale notes index {Key}", key); + } + } + } + + private async Task> ListNoteFiles(ChangelogScope notesScope, Cancel ctx) + { + var request = new ListObjectsV2Request + { + BucketName = publicBucketName, + Prefix = notesScope.Prefix + // No delimiter: list all branches recursively. + }; + + var files = new List(); + ListObjectsV2Response response; + do + { + response = await s3Client.ListObjectsV2Async(request, ctx); + foreach (var obj in response.S3Objects ?? []) + { + var relativePath = obj.Key[notesScope.Prefix.Length..]; + // Use LastIndexOf so branch names containing '/' (e.g. feature/foo) are handled correctly. + var slash = relativePath.LastIndexOf('/'); + if (slash <= 0) + continue; + var fileName = relativePath[(slash + 1)..]; + if (!IsNoteFileName(fileName)) + continue; + files.Add(obj); + _metrics.IncrementObjectsListed(); + } + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated == true); + + return files; + } + + private static bool IsNoteFileName(string fileName) => + fileName.StartsWith("note-", StringComparison.OrdinalIgnoreCase) + && (fileName.EndsWith(".yml", StringComparison.OrdinalIgnoreCase) || fileName.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase)) + && !fileName.Contains('/', StringComparison.Ordinal); + + private async Task> ExtractTargetsAsync(string key, Cancel ctx) + { + try + { + using var response = await s3Client.GetObjectAsync(new GetObjectRequest + { + BucketName = _sourceBucketName, + Key = key + }, ctx); + + await using var stream = response.ResponseStream; + using var reader = new StreamReader(stream); + var yaml = await reader.ReadToEndAsync(ctx); + + var normalized = ReleaseNotesSerialization.NormalizeYaml(yaml); + var dto = ReleaseNotesSerialization.GetEntryDeserializer().Deserialize(normalized); + + if (dto.Products is not { Count: > 0 }) + return []; + + var valid = new List(); + foreach (var target in dto.Products.Select(p => p.Target).Where(t => !string.IsNullOrWhiteSpace(t)).Distinct(StringComparer.Ordinal)) + { + if (target!.Contains('/', StringComparison.Ordinal)) + { + _logger.LogWarning("Note {Key} has target '{Target}' containing '/'; skipping — targets must be single path segments", key, target); + continue; + } + valid.Add(target); + } + return valid; + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // Note was deleted between the list and the read; skip it. + return []; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Could not read targets from note {Key}; skipping", key); + return []; + } + } + + private async Task WriteIndexAsync(string key, IReadOnlyList paths, Cancel ctx) + { + var index = new NotesIndex { Notes = paths }; + var json = JsonSerializer.Serialize(index, NotesIndexJsonContext.Default.NotesIndex); + + for (var attempt = 1; attempt <= MaxWriteAttempts; attempt++) + { + ctx.ThrowIfCancellationRequested(); + try + { + _ = await s3Client.PutObjectAsync(new PutObjectRequest + { + BucketName = publicBucketName, + Key = key, + ContentBody = json, + ContentType = "application/json" + }, ctx); + + _metrics.IncrementRegistryWrites(); + _logger.LogInformation("Wrote notes index {Key} with {Count} path(s)", key, paths.Count); + return; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + if (attempt >= MaxWriteAttempts) + throw; + + var delay = _retryBaseDelay * attempt; + _logger.LogDebug(ex, "Notes index write {Key} failed (attempt {A}/{Max}); retrying in {Delay}", key, attempt, MaxWriteAttempts, delay); + await Task.Delay(delay, ctx); + } + } + } +} diff --git a/src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs b/src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs index 64132e18be..4a65c51e2d 100644 --- a/src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs +++ b/src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs @@ -31,6 +31,7 @@ public sealed class ScrubberProcessor( IChangelogContentScrubber scrubber, BundleRegistryReconciler reconciler, ShallowRegistryReconciler shallowReconciler, + NotesIndexReconciler notesReconciler, ReconcileMetrics? metrics = null ) { @@ -75,6 +76,7 @@ public async Task> ProcessAsync(IReadOnlyList(StringComparer.Ordinal); var groupWork = new Dictionary(StringComparer.Ordinal); + var notesWork = new Dictionary(StringComparer.Ordinal); var shallowWork = new Dictionary(); var failedIds = new HashSet(StringComparer.Ordinal); @@ -87,7 +89,7 @@ public async Task> ProcessAsync(IReadOnlyList> ProcessAsync(IReadOnlyList objectWork, Dictionary groupWork, + Dictionary notesWork, Dictionary shallowWork) { var hasScope = ChangelogScope.TryFromKey(key, out var scope); @@ -170,6 +187,14 @@ private void Classify( return; } + // Notes indexes (notes-{target}.json) are reconciler-owned; a client that uploads one is + // rejected here — the reconciler writes directly to the public bucket, so no copy is needed. + if (ChangelogKeys.IsNotesIndex(key)) + { + _logger.LogWarning("Rejecting client-uploaded notes index {Key}; notes indexes are reconciler-owned", key); + return; + } + if (key.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) { _logger.LogWarning("Skipping unapproved JSON key: {Key}", key); @@ -190,9 +215,25 @@ private void Classify( if (scope!.Kind == ChangelogScopeKind.Bundle) AddGroup(groupWork, scope, messageId); + + // A note-*.yml upload triggers a notes-index reconcile for the whole repo — all targets + // whose index lists this note must be rebuilt. The Changelog scope's group is {org}/{repo}/{branch}; + // the notes scope is the two-segment {org}/{repo} prefix. + if (scope.Kind == ChangelogScopeKind.Changelog) + { + var fileName = key[scope.Prefix.Length..]; + if (IsNoteFileName(fileName)) + AddNotesGroup(notesWork, scope.Group, messageId); + } + AddShallow(shallowWork, scope, messageId); } + private static bool IsNoteFileName(string fileName) => + fileName.StartsWith("note-", StringComparison.OrdinalIgnoreCase) + && (fileName.EndsWith(".yml", StringComparison.OrdinalIgnoreCase) || fileName.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase)) + && !fileName.Contains('/', StringComparison.Ordinal); + private static void AddObject( Dictionary objectWork, string key, @@ -219,6 +260,24 @@ private static void AddGroup(Dictionary groupWork, ChangelogS _ = work.MessageIds.Add(messageId); } + private static void AddNotesGroup(Dictionary notesWork, string changelogGroup, string messageId) + { + // changelogGroup is {org}/{repo}/{branch...}; extract {org}/{repo} by taking the first two segments. + var parts = changelogGroup.Split('/'); + if (parts.Length < 2) + return; + var org = parts[0]; + var repo = parts[1]; + if (!ChangelogScope.TryCreateNotes(org, repo, out var notesScope)) + return; + if (!notesWork.TryGetValue(notesScope.Prefix, out var work)) + { + work = new GroupWork(notesScope); + notesWork[notesScope.Prefix] = work; + } + _ = work.MessageIds.Add(messageId); + } + private static void AddShallow(Dictionary shallowWork, ChangelogScope scope, string messageId) { if (!shallowWork.TryGetValue(scope.Kind, out var work)) diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/NotesIndexReconcilerTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/NotesIndexReconcilerTests.cs new file mode 100644 index 0000000000..b5a34ca9cc --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Reconciliation/NotesIndexReconcilerTests.cs @@ -0,0 +1,188 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json; +using AwesomeAssertions; +using Elastic.Changelog.Reconciliation; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Elastic.Changelog.Tests.Reconciliation; + +public class NotesIndexReconcilerTests +{ + private const string PublicBucket = "public-bucket"; + + private const string NoteYaml = + "title: Slow rollover known issue\n" + + "type: known-issue\n" + + "products:\n" + + " - product: elasticsearch\n" + + " target: 9.0.0\n"; + + private const string NoteYamlTwoTargets = + "title: Two-version known issue\n" + + "type: known-issue\n" + + "products:\n" + + " - product: elasticsearch\n" + + " target: 9.0.0\n" + + " - product: elasticsearch\n" + + " target: 9.1.0\n"; + + private readonly FakeS3 _s3 = new(PublicBucket); + private readonly NotesIndexReconciler _reconciler; + + public NotesIndexReconcilerTests() => + _reconciler = new NotesIndexReconciler( + NullLoggerFactory.Instance, _s3.Client, PublicBucket, retryBaseDelay: TimeSpan.Zero); + + private static ChangelogScope NotesScope(string org = "elastic", string repo = "elasticsearch") + { + _ = ChangelogScope.TryCreateNotes(org, repo, out var scope); + return scope!; + } + + private void SeedNote(string branch, string fileName, string yaml) => + _s3.Seed(PublicBucket, $"changelog/elastic/elasticsearch/{branch}/{fileName}", yaml); + + private NotesIndex ReadIndex(string target) => + JsonSerializer.Deserialize( + _s3.ContentOf(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", target)), + NotesIndexJsonContext.Default.NotesIndex)!; + + [Fact] + public void DirectYamlParse_NoteYaml_HasProducts() + { + var dto = ReleaseNotesSerialization.GetEntryDeserializer().Deserialize(NoteYaml); + dto.Products.Should().NotBeNullOrEmpty("YAML has products"); + dto.Products?[0].Target.Should().Be("9.0.0"); + } + + [Fact] + public async Task ReconcileRepo_SingleNote_WritesIndex() + { + SeedNote("main", "note-slow-rollover.yml", NoteYaml); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + _s3.ListCalls.Should().BeGreaterThan(0, "reconciler should have listed the bucket"); + _s3.Gets.Count.Should().BeGreaterThan(0, $"reconciler should have fetched the note; ListCalls={_s3.ListCalls} SeedExists={_s3.Exists(PublicBucket, "changelog/elastic/elasticsearch/main/note-slow-rollover.yml")}"); + _s3.Puts.Count.Should().BeGreaterThan(0, $"reconciler should have written the index; ListCalls={_s3.ListCalls} Gets={_s3.Gets.Count}"); + var index = ReadIndex("9.0.0"); + index.Notes.Should().BeEquivalentTo(["main/note-slow-rollover.yml"]); + } + + [Fact] + public async Task ReconcileRepo_NoteWithTwoTargets_AppearsInBothIndexes() + { + SeedNote("main", "note-two-targets.yml", NoteYamlTwoTargets); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + ReadIndex("9.0.0").Notes.Should().BeEquivalentTo(["main/note-two-targets.yml"]); + ReadIndex("9.1.0").Notes.Should().BeEquivalentTo(["main/note-two-targets.yml"]); + } + + [Fact] + public async Task ReconcileRepo_SameNoteNameOnTwoBranches_BothInIndex() + { + SeedNote("main", "note-slow-rollover.yml", NoteYaml); + SeedNote("9.0", "note-slow-rollover.yml", NoteYaml); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + var index = ReadIndex("9.0.0"); + index.Notes.Should().BeEquivalentTo([ + "9.0/note-slow-rollover.yml", + "main/note-slow-rollover.yml" + ]); + } + + [Fact] + public async Task ReconcileRepo_NoNotes_WritesNoIndexes() + { + // Seed a regular changelog entry that is not a note-*.yml + _s3.Seed(PublicBucket, "changelog/elastic/elasticsearch/main/12345.yaml", "title: PR entry"); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + _s3.Puts.Should().BeEmpty(); + } + + [Fact] + public async Task ReconcileRepo_NoteWithNoProducts_NotIncludedInAnyIndex() + { + SeedNote("main", "note-no-products.yml", "title: Note with no products\ntype: known-issue"); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + _s3.Puts.Should().BeEmpty(); + } + + [Fact] + public async Task ReconcileRepo_IndexPathsAreSorted() + { + SeedNote("main", "note-b.yml", NoteYaml); + SeedNote("9.0", "note-a.yml", NoteYaml); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + var index = ReadIndex("9.0.0"); + index.Notes.Should().Equal(["9.0/note-a.yml", "main/note-b.yml"]); + } + + [Fact] + public async Task ReconcileRepo_BranchWithSlashInName_IsIncludedInIndex() + { + // Branch name contains '/' — e.g. "feature/my-fix" + SeedNote("feature/my-fix", "note-slow-rollover.yml", NoteYaml); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + var index = ReadIndex("9.0.0"); + index.Notes.Should().BeEquivalentTo(["feature/my-fix/note-slow-rollover.yml"]); + } + + [Fact] + public async Task ReconcileRepo_StaleTargetRemoved_OldIndexDeleted() + { + // Pre-seed a stale notes-8.0.0.json index from a previous reconcile run. + _s3.Seed(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "8.0.0"), + /*lang=json,strict*/ + """{"notes":["old/note-stale.yml"]}"""); + + // Only seed a note for 9.0.0. + SeedNote("main", "note-slow-rollover.yml", NoteYaml); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + // 9.0.0 index should be written. + ReadIndex("9.0.0").Notes.Should().BeEquivalentTo(["main/note-slow-rollover.yml"]); + + // 8.0.0 index should have been deleted. + _s3.Deletes.Should().ContainSingle() + .Which.Key.Should().Be(ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "8.0.0")); + } + + [Fact] + public async Task ReconcileRepo_NoNotes_DeletesAllExistingIndexes() + { + // Pre-seed a stale notes index. + _s3.Seed(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "9.0.0"), + /*lang=json,strict*/ + """{"notes":["old/note-stale.yml"]}"""); + + // No note files — just an unrelated changelog entry. + _s3.Seed(PublicBucket, "changelog/elastic/elasticsearch/main/12345.yaml", "title: PR entry"); + + await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); + + // No new indexes should be written. + _s3.Puts.Should().BeEmpty(); + + // The stale index should be deleted. + _s3.Deletes.Should().ContainSingle() + .Which.Key.Should().Be(ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "9.0.0")); + } +} diff --git a/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs b/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs index 3daaa01097..bfb339b767 100644 --- a/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs +++ b/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs @@ -34,8 +34,10 @@ public ScrubberProcessorTests() NullLoggerFactory.Instance, _s3.Client, PublicBucket, retryBaseDelay: TimeSpan.Zero, metrics: _metrics); var shallowReconciler = new ShallowRegistryReconciler( NullLoggerFactory.Instance, _s3.Client, PublicBucket, retryBaseDelay: TimeSpan.Zero, metrics: _metrics); + var notesReconciler = new NotesIndexReconciler( + NullLoggerFactory.Instance, _s3.Client, PublicBucket, sourceBucketName: PrivateBucket, retryBaseDelay: TimeSpan.Zero, metrics: _metrics); _processor = new ScrubberProcessor( - NullLoggerFactory.Instance, _s3.Client, PublicBucket, _scrubber, reconciler, shallowReconciler, _metrics); + NullLoggerFactory.Instance, _s3.Client, PublicBucket, _scrubber, reconciler, shallowReconciler, notesReconciler, _metrics); } private Cancel Ctx => TestContext.Current.CancellationToken; @@ -352,6 +354,44 @@ public async Task Process_BatchMixingObjectAndRegistryEvents_MarksGroupContribut _metrics.GroupReconciles.Should().Be(1); } + [Fact] + public async Task Process_ClientUploadedNotesIndex_IsRejectedWithNoPublicWrite() + { + // The notes index is reconciler-owned; a client that uploads notes-*.json must be blocked. + _s3.Seed(PrivateBucket, "changelog/elastic/kibana/notes-9.0.0.json", /*lang=json,strict*/ """{"notes":[]}"""); + + var failed = await _processor.ProcessAsync([Message("ObjectCreated:Put", "changelog/elastic/kibana/notes-9.0.0.json")], Ctx); + + failed.Should().BeEmpty(); + _s3.Exists(PublicBucket, "changelog/elastic/kibana/notes-9.0.0.json").Should().BeFalse(); + // No reconcile triggered — only logging + _metrics.GroupReconciles.Should().Be(0); + } + + [Fact] + public async Task Process_NoteFile_ScrubbedAndNotesReconcileTriggered() + { + // language=yaml + var noteYaml = """ + title: Known rollover issue + type: known-issue + products: + - product: elasticsearch + target: 9.0.0 + """; + _s3.Seed(PrivateBucket, "changelog/elastic/elasticsearch/main/note-rollover.yml", noteYaml); + + var failed = await _processor.ProcessAsync( + [Message("ObjectCreated:Put", "changelog/elastic/elasticsearch/main/note-rollover.yml")], Ctx); + + failed.Should().BeEmpty(); + // The note was scrubbed and copied to the public bucket + _s3.ContentOf(PublicBucket, "changelog/elastic/elasticsearch/main/note-rollover.yml") + .Should().StartWith("scrubbed:"); + // The notes index was written (reconciler read the note and produced notes-9.0.0.json) + _s3.Exists(PublicBucket, "changelog/elastic/elasticsearch/notes-9.0.0.json").Should().BeTrue(); + } + // language=yaml private static string BundleYaml() => """ products: