ApiExplorer: discover op-*.md and tag-*.md supplemental files

config/assembler.yml

@@ -58,6 +58,7 @@ environments:
       enabled: false
     feature_flags:
       NAVIGATION_PREVIEW: true
+      ASSEMBLER_API_EXPLORER: true
   air-gapped:
     uri: http://localhost:8080
     path_prefix: docs

docs/_docset.yml

@@ -1,3 +1,3 @@
 project: 'doc-builder'
 max_toc_depth: 2
 dev_docs: true
@@ -38,6 +38,18 @@
 features:
   primary-nav: false
 
+# Local fixture for ApiExplorer and supplemental files. The URL key must not collide
+# with docs-content's `elasticsearch` key: assembler preview still builds this checkout
+# even though the repo is `skip: true` in assembler.yml.
+# `repository:` matches the version-index publisher; without it, lookup uses this
+# checkout (`elastic/docs-builder`) and `--strict` fails. Isolated serve (no
+# `--watch`): /api/doc/docs-builder-elasticsearch/
+api:
+  docs-builder-elasticsearch:
+    - spec: elasticsearch.json
+      product: elasticsearch
+      repository: elastic/elasticsearch-specification
+
 cta:
   docs-builder:
     button:

docs/api/docs-builder-elasticsearch/op-async-search-get.md

@@ -0,0 +1,5 @@
+# Async search get supplemental fixture
+
+This file is a local fixture for docs-builder development.
+
+Discovery should associate it with the `async-search-get` operation.

docs/data/openapi/api-explorer.md

@@ -226,15 +226,18 @@ api:
 
 ## When the API Explorer runs
 
-The API Explorer generates documentation in two scenarios:
+The API Explorer generates documentation in these scenarios:
 
 - **`docs-builder build`**: API docs are generated as part of the standard build. Use `--skip-api` to skip generation for faster iteration on content.
 - **`docs-builder serve`**: API docs are generated on startup and regenerated automatically when spec files change.
+- **Assembler builds**: API docs are generated when the `ASSEMBLER_API_EXPLORER` feature flag is on. That flag is on for the `staging` and `preview` environments. Production stays off until cutover.
 
 :::{note}
 API generation is skipped when running `docs-builder serve --watch`. This is a performance optimization for `dotnet watch` workflows. Run `serve` without `--watch` to include API docs in your local preview.
 :::
 
+This repository's own `_docset.yml` declares a local `docs-builder-elasticsearch` API that reads `elasticsearch.json` and sets `repository: elastic/elasticsearch-specification`. Use that entry to preview ApiExplorer and supplemental files during isolated `docs-builder serve` (open `/api/doc/docs-builder-elasticsearch/`). The key is not `elasticsearch`, so assembler preview does not collide with docs-content.
+
 ## Link to API pages in navigation
 
 You can reference API pages in your `toc.yml` or `docset.yml` navigation using cross-link syntax:

src/Elastic.ApiExplorer/Infrastructure/ApiUrlBuilder.cs

@@ -42,11 +42,14 @@ public static string OperationMoniker(string? operationId, string route)
 	public static string SchemaMoniker(string schemaId) =>
 		schemaId.Replace('.', '-').ToLowerInvariant();
 
-	/// <summary>Deterministic URL leaf for <c>.../group/{segment}</c> from the canonical tag name.</summary>
-	public static string TagMoniker(string? tagName)
+	/// <summary>
+	/// URL slug for a tag, without the <c>endpoint-</c> prefix. Spaces become hyphens and the
+	/// result is lowercased; underscores are kept. Empty or whitespace names become <c>unknown</c>.
+	/// </summary>
+	public static string TagSlug(string? tagName)
 	{
 		if (string.IsNullOrWhiteSpace(tagName))
-			return "endpoint-unknown";
+			return "unknown";
 
 		var s = tagName.Trim();
 		s = string.Join(" ", s.Split(' ', StringSplitOptions.RemoveEmptyEntries));
@@ -56,12 +59,12 @@ public static string TagMoniker(string? tagName)
 		s = s.Replace("/", "-", StringComparison.Ordinal);
 		s = s.Replace(" ", "-", StringComparison.Ordinal);
 		s = s.ToLowerInvariant();
-		if (string.IsNullOrEmpty(s))
-			return "endpoint-unknown";
-
-		return $"endpoint-{s}";
+		return string.IsNullOrEmpty(s) ? "unknown" : s;
 	}
 
+	/// <summary>Deterministic URL leaf for <c>.../group/{segment}</c> from the canonical tag name.</summary>
+	public static string TagMoniker(string? tagName) => $"endpoint-{TagSlug(tagName)}";
+
 	[GeneratedRegex(@"\s*\(([^)]+)\)")]
 	private static partial Regex ParentheticalSuffixPattern();
 }

src/Elastic.ApiExplorer/OpenApiGenerator.cs

@@ -9,6 +9,7 @@
 using Elastic.ApiExplorer.Model;
 using Elastic.ApiExplorer.Navigation;
 using Elastic.ApiExplorer.Operations;
+using Elastic.ApiExplorer.Supplemental;
 using Elastic.Documentation;
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Products;
@@ -228,6 +229,7 @@ private async Task GenerateApiProduct(
 		IReadOnlyList<ApiVersionSwitcherItem> versionSwitcherItems,
 		Cancel ctx)
 	{
+		_ = DiscoverSupplemental(openApiDocument, apiConfig);
 		var navigation = CreateNavigation(prefix, openApiDocument, apiConfig);
 		_logger.LogInformation("Generating OpenApiDocument {Title}", openApiDocument.Info?.Title ?? "<no title>");
 
@@ -245,6 +247,27 @@ private async Task GenerateApiProduct(
 		await RenderNavigationItems(renderContext, navigationRenderer, navigation, ctx).ConfigureAwait(false);
 	}
 
+	/// <summary>
+	/// Associates <c>op-*.md</c> / <c>tag-*.md</c> files under <c>api/&lt;key&gt;/</c> with this
+	/// document. Merge into page models is a later change; this call logs and exposes the result.
+	/// </summary>
+	internal ApiSupplementalDiscoveryResult DiscoverSupplemental(
+		OpenApiDocument openApiDocument,
+		ResolvedApiConfiguration? apiConfig)
+	{
+		var result = ApiSupplementalDiscovery.Discover(apiConfig?.ApiContentDirectory, openApiDocument);
+		if (result.Operations.Count == 0 && result.Tags.Count == 0 && result.Unmatched.Count == 0)
+			return result;
+
+		_logger.LogInformation(
+			"API '{ApiKey}' supplemental files: {Operations} operations, {Tags} tags, {Unmatched} unmatched",
+			apiConfig?.ProductKey ?? "unknown",
+			result.Operations.Count,
+			result.Tags.Count,
+			result.Unmatched.Count);
+		return result;
+	}
+
 	private async Task RenderNavigationItems(
 		ApiRenderContext renderContext,
 		IsolatedBuildNavigationHtmlWriter navigationRenderer,

src/Elastic.ApiExplorer/Supplemental/ApiSupplementalDiscovery.cs

@@ -0,0 +1,182 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using Elastic.ApiExplorer.Infrastructure;
+using Microsoft.OpenApi;
+
+namespace Elastic.ApiExplorer.Supplemental;
+
+public sealed record TagSlugCollision(string Slug, IReadOnlyList<string> TagNames);
+
+public sealed record ApiSupplementalVersionedFile(IFileInfo File, ApiSupplementalFileName Name);
+
+public sealed class ApiSupplementalDiscoveryResult
+{
+	public required IReadOnlyDictionary<string, IFileInfo> Operations { get; init; }
+	public required IReadOnlyDictionary<string, IFileInfo> Tags { get; init; }
+	public required IReadOnlyList<IFileInfo> Unmatched { get; init; }
+	public required IReadOnlyList<IFileInfo> Ignored { get; init; }
+	public required IReadOnlyList<ApiSupplementalVersionedFile> VersionSuffixed { get; init; }
+	public required IReadOnlyList<TagSlugCollision> TagSlugCollisions { get; init; }
+}
+
+/// <summary>
+/// Discovers top-level <c>op-*.md</c> / <c>tag-*.md</c> files under <c>api/&lt;key&gt;/</c>.
+/// Does not emit diagnostics; unmatched convention files are returned for later validation.
+/// </summary>
+public static class ApiSupplementalDiscovery
+{
+	public static ApiSupplementalDiscoveryResult Discover(
+		IDirectoryInfo? folder,
+		IReadOnlyCollection<string> operationIds,
+		IReadOnlyCollection<string> tagNames)
+	{
+		var (tagBySlug, collisions) = IndexTags(tagNames);
+		return MatchFiles(folder, operationIds.ToHashSet(StringComparer.Ordinal), tagBySlug, collisions);
+	}
+
+	public static ApiSupplementalDiscoveryResult Discover(IDirectoryInfo? folder, OpenApiDocument document)
+	{
+		var (operationIds, tagNames) = CollectEntities(document);
+		var (tagBySlug, collisions) = IndexTags(tagNames);
+		return MatchFiles(folder, operationIds, tagBySlug, collisions);
+	}
+
+	private static ApiSupplementalDiscoveryResult MatchFiles(
+		IDirectoryInfo? folder,
+		HashSet<string> operationIds,
+		Dictionary<string, string> tagBySlug,
+		IReadOnlyList<TagSlugCollision> collisions)
+	{
+		if (folder is null || !folder.Exists)
+			return NoFiles(collisions);
+
+		var operations = new Dictionary<string, IFileInfo>(StringComparer.Ordinal);
+		var tags = new Dictionary<string, IFileInfo>(StringComparer.Ordinal);
+		var unmatched = new List<IFileInfo>();
+		var ignored = new List<IFileInfo>();
+		var versionSuffixed = new List<ApiSupplementalVersionedFile>();
+
+		foreach (var file in folder.EnumerateFiles("*.md"))
+		{
+			if (!ApiSupplementalName.TryParse(file.Name, out var name))
+			{
+				ignored.Add(file);
+				continue;
+			}
+
+			if (name.IsVersionSuffixed)
+			{
+				versionSuffixed.Add(new ApiSupplementalVersionedFile(file, name));
+				continue;
+			}
+
+			if (name.Kind == ApiSupplementalKind.Operation)
+			{
+				if (operationIds.Contains(name.Stem) && operations.TryAdd(name.Stem, file))
+					continue;
+				unmatched.Add(file);
+				continue;
+			}
+
+			if (tagBySlug.TryGetValue(name.Stem, out var tagName) && tags.TryAdd(tagName, file))
+				continue;
+
+			unmatched.Add(file);
+		}
+
+		return new ApiSupplementalDiscoveryResult
+		{
+			Operations = operations,
+			Tags = tags,
+			Unmatched = unmatched,
+			Ignored = ignored,
+			VersionSuffixed = versionSuffixed,
+			TagSlugCollisions = collisions
+		};
+	}
+
+	private static (HashSet<string> OperationIds, HashSet<string> TagNames) CollectEntities(OpenApiDocument document)
+	{
+		var operations = new HashSet<string>(StringComparer.Ordinal);
+		var tags = new HashSet<string>(StringComparer.Ordinal);
+
+		if (document.Tags is not null)
+		{
+			foreach (var tag in document.Tags)
+			{
+				if (!string.IsNullOrEmpty(tag.Name))
+					_ = tags.Add(tag.Name);
+			}
+		}
+
+		foreach (var path in document.Paths ?? [])
+		{
+			if (path.Value.Operations is null)
+				continue;
+
+			foreach (var operation in path.Value.Operations.Values)
+			{
+				if (!string.IsNullOrWhiteSpace(operation.OperationId))
+					_ = operations.Add(operation.OperationId);
+
+				if (operation.Tags is null)
+					continue;
+
+				foreach (var tagRef in operation.Tags)
+				{
+					var name = OperationTagName(tagRef);
+					if (!string.IsNullOrEmpty(name))
+						_ = tags.Add(name);
+				}
+			}
+		}
+
+		return (operations, tags);
+	}
+
+	private static string? OperationTagName(OpenApiTagReference tagRef) =>
+		!string.IsNullOrEmpty(tagRef.Name) ? tagRef.Name : tagRef.Reference?.Id;
+
+	private static (Dictionary<string, string> UniqueBySlug, IReadOnlyList<TagSlugCollision> Collisions) IndexTags(
+		IReadOnlyCollection<string> tagNames)
+	{
+		var bySlug = new Dictionary<string, List<string>>(StringComparer.Ordinal);
+		foreach (var tagName in tagNames)
+		{
+			var slug = ApiUrlBuilder.TagSlug(tagName);
+			if (!bySlug.TryGetValue(slug, out var names))
+			{
+				names = [];
+				bySlug[slug] = names;
+			}
+
+			if (!names.Contains(tagName, StringComparer.Ordinal))
+				names.Add(tagName);
+		}
+
+		var unique = new Dictionary<string, string>(StringComparer.Ordinal);
+		var collisions = new List<TagSlugCollision>();
+		foreach (var (slug, names) in bySlug)
+		{
+			if (names.Count == 1)
+				unique[slug] = names[0];
+			else
+				collisions.Add(new TagSlugCollision(slug, names));
+		}
+
+		return (unique, collisions);
+	}
+
+	private static ApiSupplementalDiscoveryResult NoFiles(IReadOnlyList<TagSlugCollision> collisions) => new()
+	{
+		Operations = new Dictionary<string, IFileInfo>(),
+		Tags = new Dictionary<string, IFileInfo>(),
+		Unmatched = [],
+		Ignored = [],
+		VersionSuffixed = [],
+		TagSlugCollisions = collisions
+	};
+}

src/Elastic.ApiExplorer/Supplemental/ApiSupplementalName.cs

@@ -0,0 +1,57 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text.RegularExpressions;
+
+namespace Elastic.ApiExplorer.Supplemental;
+
+public enum ApiSupplementalKind
+{
+	Operation,
+	Tag
+}
+
+public readonly record struct ApiSupplementalFileName(
+	ApiSupplementalKind Kind,
+	string Stem,
+	int? VersionMajor)
+{
+	public bool IsVersionSuffixed => VersionMajor is not null;
+}
+
+/// <summary>
+/// Parses <c>op-*.md</c> / <c>tag-*.md</c> filenames. Operation stems are the spec
+/// <c>operationId</c> with no rewriting. Tag stems are <see cref="ApiUrlBuilder.TagSlug"/>.
+/// </summary>
+public static partial class ApiSupplementalName
+{
+	public static bool TryParse(string fileName, out ApiSupplementalFileName parsed)
+	{
+		parsed = default;
+		var match = FileNamePattern().Match(fileName);
+		if (!match.Success)
+			return false;
+
+		var kind = match.Groups[1].Value.Equals("op", StringComparison.OrdinalIgnoreCase)
+			? ApiSupplementalKind.Operation
+			: ApiSupplementalKind.Tag;
+		var stem = match.Groups[2].Value;
+		if (stem.Length == 0)
+			return false;
+
+		int? version = null;
+		if (match.Groups[3].Success)
+		{
+			if (!int.TryParse(match.Groups[3].Value, out var major))
+				return false;
+			version = major;
+		}
+
+		parsed = new ApiSupplementalFileName(kind, stem, version);
+		return true;
+	}
+
+	[GeneratedRegex(@"^(op|tag)-(.+?)(?:\.v(\d+))?\.md$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)]
+	private static partial Regex FileNamePattern();
+}