Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public record ChangelogConfiguration
/// Filename strategy for generated changelog files.
/// Controls how files created by 'changelog add' are named.
/// </summary>
public FilenameStrategy Filename { get; init; } = FilenameStrategy.Timestamp;
public FilenameStrategy Filename { get; init; } = FilenameStrategy.Pr;

/// <summary>
/// Bundle configuration with profiles and defaults.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -303,15 +303,27 @@ internal static ChangelogConfigurationYaml DeserializeConfiguration(string yaml)
};

// Process filename strategy
var filenameStrategy = FilenameStrategy.Timestamp;
var filenameStrategy = FilenameStrategy.Pr;
if (!string.IsNullOrWhiteSpace(yamlConfig.Filename))
{
if (!FilenameStrategyExtensions.TryParse(yamlConfig.Filename, out var parsed, ignoreCase: true, allowMatchingMetadataAttribute: true))
{
var valid = string.Join(", ", FilenameStrategyExtensions.GetValues().Select(v => v.ToStringFast(true)));
collector.EmitError(configPath, $"filename: '{yamlConfig.Filename}' is not valid. Use one of: {valid}");
collector.EmitError(configPath, $"filename: '{yamlConfig.Filename}' is not valid. The only supported value is 'pr'. Changelog entries are keyed by PR number; for items with no PR use 'changelog note'.");
return null;
}

if (parsed == FilenameStrategy.Timestamp)
{
collector.EmitError(configPath, "filename: 'timestamp' is no longer supported. Changelog entries are keyed by PR number; for items with no PR use 'changelog note'.");
return null;
}

if (parsed == FilenameStrategy.Issue)
{
collector.EmitError(configPath, "filename: 'issue' is no longer supported. Changelog entries are keyed by PR number; for items with no PR use 'changelog note'.");
return null;
}

filenameStrategy = parsed;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,119 @@ public async Task<IReadOnlyList<CdnChangelogEntry>> FetchAsync(
return entries;
}

/// <summary>
/// Fetches a named set of changelog entries directly from the CDN without consulting the pool registry.
/// Each entry is fetched by its key (<c>{base}/changelog/{org}/{repo}/{branch}/{fileName}</c>); a 404
/// is a hard error (the caller explicitly requested the entry), while 5xx / transport errors use the
/// normal retry budget. Returns <c>null</c> after emitting an error when any entry cannot be fetched.
/// </summary>
public async Task<IReadOnlyList<CdnChangelogEntry>?> FetchNamedAsync(
Uri baseUri,
string org,
string repo,
string branch,
IReadOnlyList<string> fileNames,
Action<string> emitError,
Cancel ctx)
{
var poolLabel = $"{org}/{repo}/{branch}";

if (!ChangelogKeys.IsValidOrg(org) || !ChangelogKeys.IsValidRepo(repo) || !ChangelogKeys.IsValidBranch(branch))
{
emitError(
$"Invalid changelog pool '{poolLabel}': the org, repo, and each '/'-delimited branch segment must be non-empty ASCII letters, digits, '.', '_' or '-' (org allows only letters, digits and '-') and must not be '.' or '..'.");
return null;
}

var poolSegments = ChangelogKeys.PoolSegments(org, repo, branch);
var entries = new List<CdnChangelogEntry>(fileNames.Count);
var hasError = false;

foreach (var fileName in fileNames)
{
ctx.ThrowIfCancellationRequested();

if (!ChangelogKeys.IsSafeFileName(fileName))
{
emitError($"Requested changelog entry '{fileName}' is not a valid file name for pool '{poolLabel}'.");
hasError = true;
continue;
}

var entryUri = CombineSegments(baseUri, [.. poolSegments, fileName]);
var (fetched, content, lastError) = await TryFetchNamedEntryAsync(entryUri, fileName, poolLabel, ctx).ConfigureAwait(false);
if (fetched)
{
entries.Add(new CdnChangelogEntry(fileName, content));
continue;
}

// Explicit path-list request: a miss is a pipeline error (wrong name, not uploaded, or renamed).
emitError(
$"Changelog entry '{fileName}' for '{poolLabel}' could not be fetched from {entryUri}: {lastError}. " +
"Ensure the entry was uploaded (changelog upload), or pass --force-local / --directory to bundle local files instead.");
hasError = true;
}

return hasError ? null : entries;
}

/// <summary>
/// Fetches a single explicitly-requested entry. A 404 is surfaced immediately (not retried) because the
/// caller knows the entry should exist; 5xx and transport errors use the normal retry budget.
/// Permanent client errors (4xx other than 404) also fail immediately without retry.
/// </summary>
private async Task<(bool Fetched, string Content, string? LastError)> TryFetchNamedEntryAsync(Uri uri, string fileName, string poolLabel, Cancel ctx)
{
string? lastError = null;

for (var attempt = 1; attempt <= _maxAttempts; attempt++)
{
ctx.ThrowIfCancellationRequested();
try
{
var (notFound, content) = await FetchTextOrNotFoundAsync(uri, attempt, ctx).ConfigureAwait(false);
if (notFound)
return (false, string.Empty, "404 Not Found — entry does not exist in the pool");
if (attempt > 1)
_logger.LogInformation("Fetched changelog entry '{File}' for {Pool} on attempt {Attempt}/{Max}", fileName, poolLabel, attempt, _maxAttempts);
return (true, content, null);
}
catch (Exception ex) when (ex is not OperationCanceledException)
Comment thread
Mpdreamz marked this conversation as resolved.
{
// Permanent client errors (4xx — 404 is handled above as notFound) must not be retried.
if (ex is HttpRequestException { StatusCode: >= HttpStatusCode.BadRequest and < HttpStatusCode.InternalServerError })
return (false, string.Empty, ex.Message);

lastError = ex.Message;
if (attempt >= _maxAttempts)
break;

var delay = RetryDelay(attempt);
_logger.LogDebug(
"Changelog entry '{File}' for {Pool} not yet available (attempt {Attempt}/{Max}: {Error}); retrying in {Delay}",
fileName, poolLabel, attempt, _maxAttempts, ex.Message, delay);
await _sleep(delay, ctx).ConfigureAwait(false);
}
}

return (false, string.Empty, lastError);
}

/// <summary>Returns (notFound: true) for a 404; throws for other non-success status codes.</summary>
private async Task<(bool NotFound, string Content)> FetchTextOrNotFoundAsync(Uri uri, int attempt, Cancel ctx)
{
var requestUri = attempt > 1 ? WithCacheBuster(uri) : uri;
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
if (attempt > 1)
_ = request.Headers.TryAddWithoutValidation("Cache-Control", "no-cache");
using var response = await _httpClient.SendAsync(request, ctx).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound)
return (true, string.Empty);
_ = response.EnsureSuccessStatusCode();
return (false, await response.Content.ReadAsStringAsync(ctx).ConfigureAwait(false));
}

/// <summary>
/// Fetches a single entry, retrying transient failures (most importantly a not-yet-propagated 404)
/// up to <see cref="_maxAttempts"/> times with exponential backoff. Retry requests are cache-busted
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,17 @@ public async Task<bool> BundleChangelogs(IDiagnosticsCollector collector, Bundle
if (!ValidateInput(collector, input, requireDirectoryExists: !useCdn))
return false;

// --all and --input-products require reading every entry body; that is only possible locally.
// On the CDN path entries are probed by key, so there is nothing to enumerate without a PR list.
if (useCdn && (input.All || input.InputProducts is { Count: > 0 }))
{
var flag = input.All ? "--all" : "--input-products";
collector.EmitError(string.Empty,
$"{flag} is not supported when sourcing changelog entries from the CDN, because entries are fetched by key (one per PR) and there is no pool enumeration. " +
"Pass --force-local or --directory to bundle from a local checkout instead.");
return false;
}

if (!ValidatePlaceholderUsage(collector, input))
return false;

Expand Down Expand Up @@ -357,21 +368,25 @@ public async Task<bool> BundleChangelogs(IDiagnosticsCollector collector, Bundle
}
else if (useCdn)
{
var contents = await FetchCdnEntriesAsync(collector, authoringOwner, authoringRepo, authoringBranch, ctx);
if (contents == null)
return false;
if (requestedEntryNames is not null)
{
var poolLabel = $"{authoringOwner}/{authoringRepo}/{authoringBranch}";
var selected = SelectRequestedCdnEntries(collector, contents, requestedEntryNames, poolLabel);
// --files on the CDN path: fetch each entry directly by key; no registry needed.
var selected = await FetchCdnNamedEntriesAsync(collector, authoringOwner, authoringRepo, authoringBranch, requestedEntryNames, ctx);
if (selected == null)
return false;
_logger.LogInformation("Matching {Count} explicitly selected changelog entries from the CDN", selected.Count);
var filesCriteria = filterCriteria with { IncludeAll = true };
matchResult = entryMatcher.MatchChangelogContents(collector, selected, filesCriteria, ctx);
}
else
{
// --prs / --issues on the CDN path: still uses the pool registry for now (Step 9 will
// switch this to per-PR probing once canonical keys and markers are in place).
var contents = await FetchCdnEntriesAsync(collector, authoringOwner, authoringRepo, authoringBranch, ctx);
if (contents == null)
return false;
matchResult = entryMatcher.MatchChangelogContents(collector, contents, filterCriteria, ctx);
}
}
else
{
Expand Down Expand Up @@ -1096,6 +1111,55 @@ private BundleChangelogsArguments ApplyConfigDefaults(BundleChangelogsArguments
return null;
}

/// <summary>
/// Fetches a named list of changelog entries directly from the CDN without consulting the pool registry.
/// Used for the <c>--files</c> CDN path. Returns <c>null</c> after emitting an error on any failure.
/// </summary>
private async Task<IReadOnlyList<(string FileName, string Content)>?> FetchCdnNamedEntriesAsync(
IDiagnosticsCollector collector,
string? org,
string? repo,
string? branch,
IReadOnlyList<string> fileNames,
Cancel ctx)
{
if (string.IsNullOrWhiteSpace(repo))
{
collector.EmitError(string.Empty,
"Sourcing changelog entries from the CDN requires a resolvable authoring repository. " +
"Set bundle.repo in changelog.yml (or pass --repo), or pass --force-local / --directory to bundle local files.");
return null;
}

var resolvedOrg = string.IsNullOrWhiteSpace(org) ? DefaultOwner : org;
var resolvedBranch = string.IsNullOrWhiteSpace(branch) ? DefaultBranch : branch;

var baseUri = ChangelogCdn.ResolveBaseUri();
if (baseUri is null)
{
collector.EmitError(string.Empty,
$"No valid changelog CDN base URL is configured. Set the {ChangelogCdn.BaseUrlEnvironmentVariable} environment variable to an absolute http(s) URL.");
return null;
}

var entries = await _entryFetcher.FetchNamedAsync(
baseUri,
resolvedOrg,
repo,
resolvedBranch,
fileNames,
msg => collector.EmitError(string.Empty, msg),
ctx);

if (entries == null)
return null;

_logger.LogInformation("Fetched {Count} named changelog entry(ies) for {Pool} from CDN",
entries.Count, $"{resolvedOrg}/{repo}/{resolvedBranch}");

return entries.Select(e => (e.FileName, e.Content)).ToList();
}

/// <summary>Downloads the authoring <paramref name="org"/>/<paramref name="repo"/>/<paramref name="branch"/> pool's changelog entries from the CDN (<c>changelog/{org}/{repo}/{branch}/...</c>); returns null after emitting an error on any fatal fetch failure.</summary>
private async Task<IReadOnlyList<(string FileName, string Content)>?> FetchCdnEntriesAsync(
IDiagnosticsCollector collector,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ public record CreateChangelogArguments
public string? Output { get; init; }
public string? Config { get; init; }
public bool UsePrNumber { get; init; }
public bool UseIssueNumber { get; init; }
public bool? StripTitlePrefix { get; init; }
/// <summary>
/// Whether to extract release note text from PR/issue descriptions for the entry description. null = use config default.
Expand Down Expand Up @@ -147,26 +146,15 @@ public async Task<bool> CreateChangelog(IDiagnosticsCollector collector, CreateC
}
}

internal static CreateChangelogArguments ApplyConfigDefaults(CreateChangelogArguments input, ChangelogConfiguration config)
{
var usePrNumber = input.UsePrNumber;
var useIssueNumber = input.UseIssueNumber;

if (!usePrNumber && !useIssueNumber)
{
usePrNumber = config.Filename == FilenameStrategy.Pr;
useIssueNumber = config.Filename == FilenameStrategy.Issue;
}

return input with
internal static CreateChangelogArguments ApplyConfigDefaults(CreateChangelogArguments input, ChangelogConfiguration config) =>
// Filename strategy is always Pr now; UsePrNumber is kept for backward compat but is effectively always true.
input with
{
ExtractReleaseNotes = input.ExtractReleaseNotes ?? config.Extract.ReleaseNotes,
ExtractIssues = input.ExtractIssues ?? config.Extract.Issues,
StripTitlePrefix = input.StripTitlePrefix ?? config.Extract.StripTitlePrefix,
UsePrNumber = usePrNumber,
UseIssueNumber = useIssueNumber
UsePrNumber = true
};
}

/// <summary>
/// Infers products from configuration defaults or repository name.
Expand Down
49 changes: 11 additions & 38 deletions src/services/Elastic.Changelog/Creation/ChangelogFileWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,11 @@ public async Task<bool> WriteChangelogAsync(
if (!fileSystem.Directory.Exists(outputDir))
_ = fileSystem.Directory.CreateDirectory(outputDir);

// Generate filename
// Generate filename — returns null and emits an error when no PR number is derivable.
var filename = GenerateFilename(collector, input);
if (filename == null)
return false;

var filePath = fileSystem.Path.Join(outputDir, filename);

// Write UTF-8 text without BOM using explicit encoding instance.
Expand All @@ -62,9 +65,9 @@ public async Task<bool> WriteChangelogAsync(
/// <summary>Maximum filename length before extension to avoid filesystem path-too-long errors.</summary>
private const int MaxFilenameLength = 200;

private string GenerateFilename(IDiagnosticsCollector collector, CreateChangelogArguments input)
private string? GenerateFilename(IDiagnosticsCollector collector, CreateChangelogArguments input)
{
if (input.UsePrNumber && input.Prs is { Length: > 0 })
if (input.Prs is { Length: > 0 })
{
var numbers = input.Prs
.Select(pr => ChangelogTextUtilities.ExtractPrNumber(pr, input.Owner, input.Repo))
Expand All @@ -82,43 +85,13 @@ private string GenerateFilename(IDiagnosticsCollector collector, CreateChangelog
// Too many PRs: use compact format to avoid path-too-long errors
return $"{numbers[0]}-to-{numbers[^1]}-{numbers.Count}-prs.yaml";
}

collector.EmitWarning(string.Empty, $"Failed to extract PR numbers from PRs. Falling back to timestamp-based filename.");
}

if (input.UseIssueNumber && input.Issues is { Length: > 0 })
{
var numbers = input.Issues
.Select(issue => ChangelogTextUtilities.ExtractIssueNumber(issue, input.Owner, input.Repo))
.Where(n => n.HasValue)
.Select(n => n!.Value)
.Distinct()
.OrderBy(n => n)
.ToList();

if (numbers.Count > 0)
{
var joined = $"{string.Join("-", numbers)}.yaml";
if (joined.Length <= MaxFilenameLength + 5)
return joined;
return $"{numbers[0]}-to-{numbers[^1]}-{numbers.Count}-issues.yaml";
}

collector.EmitWarning(string.Empty, "Failed to extract issue numbers from issues. Falling back to timestamp-based filename.");
}

// Default: timestamp-slug.yaml
var timestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var firstPr = input.Prs is { Length: > 0 } ? input.Prs[0] : null;
var firstIssue = input.Issues is { Length: > 0 } ? input.Issues[0] : null;
var slug = string.IsNullOrWhiteSpace(input.Title)
? firstPr != null
? $"pr-{firstPr.Replace("/", "-").Replace(":", "-")}"
: firstIssue != null
? $"issue-{firstIssue.Replace("/", "-").Replace(":", "-")}"
: "changelog"
: ChangelogTextUtilities.SanitizeFilename(input.Title);
return $"{timestamp}-{slug}.yaml";
collector.EmitError(string.Empty,
"Could not derive a PR number from the provided --prs values. " +
"Changelog entries must be anchored to a PR number. " +
"For items with no PR use 'changelog note' instead.");
return null;
}

private static ChangelogEntry BuildChangelogData(CreateChangelogArguments input)
Expand Down
12 changes: 6 additions & 6 deletions src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -158,15 +158,15 @@ private void Classify(
if (!hasScope)
return;

// The two trees part ways here. Bundle manifests are reconciler-owned: the event only
// schedules a group reconcile, so client-authored JSON never reaches the public bucket
// for the tree consumers enumerate. Pool manifests stay client-authored pass-through —
// `changelog bundle` still enumerates a pool through its manifest today, and 404-probing
// only works once entries are guaranteed one-per-PR — until Phase 3 retires them.
// Bundle manifests are reconciler-owned: the event schedules a group reconcile so
// client-authored JSON never reaches the public bucket directly. Pool registry keys
// (changelog/{org}/{repo}/{branch}/registry.json) are no longer written by any client
// — the pool index was retired in #3760. Drop them with a debug log; a stale event
// from an old client is harmless and does not need to copy anything.
if (scope!.Kind == ChangelogScopeKind.Bundle)
AddGroup(groupWork, scope, messageId);
else
AddObject(objectWork, key, sourceBucket, messageId, passThrough: true);
_logger.LogDebug("Ignoring retired pool registry key: {Key}", key);
return;
}

Expand Down
Loading
Loading