From fe9c5337278ff89b293f62660b6f75de80776e6a Mon Sep 17 00:00:00 2001 From: ShaneK Date: Fri, 18 Sep 2026 11:48:13 -0700 Subject: [PATCH 1/4] feat(llms-txt): generate llms.txt and per-page markdown twins at build time --- cspell-wordlist.txt | 3 + docusaurus.config.js | 5 + plugins/docusaurus-plugin-llms-txt/README.md | 68 +++ plugins/docusaurus-plugin-llms-txt/index.js | 158 +++++++ .../lib/llms-txt.js | 180 ++++++++ .../lib/llms-txt.test.js | 217 ++++++++++ .../lib/markdown-twins.js | 356 ++++++++++++++++ .../lib/markdown-twins.test.js | 395 ++++++++++++++++++ .../docusaurus-plugin-llms-txt/lib/paths.js | 45 ++ .../lib/paths.test.js | 74 ++++ .../lib/playground-code.js | 174 ++++++++ .../lib/playground-code.test.js | 124 ++++++ 12 files changed, 1799 insertions(+) create mode 100644 plugins/docusaurus-plugin-llms-txt/README.md create mode 100644 plugins/docusaurus-plugin-llms-txt/index.js create mode 100644 plugins/docusaurus-plugin-llms-txt/lib/llms-txt.js create mode 100644 plugins/docusaurus-plugin-llms-txt/lib/llms-txt.test.js create mode 100644 plugins/docusaurus-plugin-llms-txt/lib/markdown-twins.js create mode 100644 plugins/docusaurus-plugin-llms-txt/lib/markdown-twins.test.js create mode 100644 plugins/docusaurus-plugin-llms-txt/lib/paths.js create mode 100644 plugins/docusaurus-plugin-llms-txt/lib/paths.test.js create mode 100644 plugins/docusaurus-plugin-llms-txt/lib/playground-code.js create mode 100644 plugins/docusaurus-plugin-llms-txt/lib/playground-code.test.js diff --git a/cspell-wordlist.txt b/cspell-wordlist.txt index 13b3252d86..a66d66fb1e 100644 --- a/cspell-wordlist.txt +++ b/cspell-wordlist.txt @@ -43,6 +43,7 @@ fortawesome frontmatter fullscreen geolocation +headerless iconset interactives isopen @@ -53,6 +54,8 @@ jsdelivr keyframes keytool lifecycles +llms +llmstxt localstorage mobileweb phablet diff --git a/docusaurus.config.js b/docusaurus.config.js index f020c84229..a98e562c93 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -404,8 +404,13 @@ module.exports = { 'docusaurus-plugin-copy-page-button', { injectButton: false, + // docusaurus-plugin-llms-txt writes the markdown twins instead, reusing + // this package's converter after repairing the HTML it is given. + // Turning both on would have the two race for the same files. + generateMarkdownRoutes: false, }, ], + path.resolve(__dirname, 'plugins', 'docusaurus-plugin-llms-txt'), ], customFields: {}, themes: [], diff --git a/plugins/docusaurus-plugin-llms-txt/README.md b/plugins/docusaurus-plugin-llms-txt/README.md new file mode 100644 index 0000000000..b67df3fc23 --- /dev/null +++ b/plugins/docusaurus-plugin-llms-txt/README.md @@ -0,0 +1,68 @@ +# docusaurus-plugin-llms-txt + +Writes `llms.txt` into the build output so it is served at +https://ionicframework.com/docs/llms.txt, in the format described at +[llmstxt.org](https://llmstxt.org). + +The plugin also writes a markdown twin of every docs page, which is what the +index links to. Twins are written for every version, so a v8 page is readable +as markdown too, but `llms.txt` itself covers only the current version in +English. + +## Why the twins are written here + +The conversion comes from `docusaurus-plugin-copy-page-button`, which is +already a dependency. Its own `generateMarkdownRoutes` option writes the same +files, and is deliberately left off in `docusaurus.config.js`, because plugin +`postBuild` hooks run concurrently under `Promise.all` and having both write +the same paths would be a race. The converter is reused here instead, with the +HTML repaired on the way through. + +Docusaurus emits minified HTML with the optional `` and `` end tags +left out, which that converter's parser does not account for, so every table +used to collapse onto a single line. Separately, a `` mounts its +editor on the client, so the server-rendered HTML is an empty shell and the +code examples went missing. Those snippets are on disk under `static/usage/`, +so they get read from there and spliced back in. The smaller repairs are +commented in `lib/markdown-twins.js`. + +There's one trap if you touch the path handling. The converter also has a +client-side `getMarkdownRouteUrl` that disagrees with what it writes to disk +for the site root, giving `/docs.md` where the file is `/docs/index.md`. Use +`lib/paths.js`, which follows the file on disk. + +## Which pages are covered + +Sections mirror the top-level categories of the `docs` sidebar. The generated +reference pages (the `api`, `cli` and `native` sidebars) go under +`## Optional`, the spec's reserved heading for links an agent can skip when it +needs a shorter context. + +A page is included when some sidebar points at it. That rule leaves out +`developer-resources/*`, which `vercel.json` redirects off the docs site, the +`docs/test/*` scratch pages and the orphaned `intro/first-app`, and it keeps +working as pages come and go. Draft and unlisted pages are dropped too. + +The Japanese build is skipped. It gets its own `build/ja` output root so there +is no clash with the English file, but the section labels come from the +English sidebar and nothing would link the result. + +## Descriptions + +Bullet descriptions come from the docs plugin's resolved `description`. Almost +no page sets one in frontmatter, so in practice this is Docusaurus's body +excerpt, which for most pages is the SEO title out of the in-body `` +block and reads well enough. A few fall through to something useless, and +`cleanDescription` drops those so the bullet ends up title-only. Setting a +frontmatter `description` on a page beats the excerpt. + +## Layout and tests + +```bash +npx vitest run plugins/docusaurus-plugin-llms-txt +``` + +The `index.js` hook owns the filesystem and everything under `lib/` is pure. +`llms-txt.js` builds the index, `markdown-twins.js` the twins, +`playground-code.js` reads a usage folder, and `paths.js` maps a permalink to +files and URLs. diff --git a/plugins/docusaurus-plugin-llms-txt/index.js b/plugins/docusaurus-plugin-llms-txt/index.js new file mode 100644 index 0000000000..1019e8f342 --- /dev/null +++ b/plugins/docusaurus-plugin-llms-txt/index.js @@ -0,0 +1,158 @@ +const fs = require('fs'); +const path = require('path'); + +const { buildSections, getReferencedDocIds, renderLlmsTxt } = require('./lib/llms-txt'); +const { buildTwin, hasPlaygrounds, readUsageDirs } = require('./lib/markdown-twins'); +const { toMarkdownPath, toOutputPaths } = require('./lib/paths'); + +const DOCS_PLUGIN_NAME = 'docusaurus-plugin-content-docs'; +const DOCS_PLUGIN_ID = 'default'; +const CURRENT_VERSION = 'current'; + +const INTRO = + 'Every page below links to its markdown source, and each one opens with the URL of the page it came from. ' + + 'These pages document the current version of Ionic Framework in English.'; + +/** Names the other versions that also have twins, so they are discoverable. */ +const olderVersionsNote = (loadedVersions) => { + const paths = loadedVersions + .filter((version) => version.versionName !== CURRENT_VERSION) + .map((version) => `${version.path.replace(/\/+$/, '')}/`); + + return paths.length > 0 ? ` Pages for earlier versions are at the same paths under ${paths.join(' and ')}.` : ''; +}; + +const firstExisting = (candidates) => candidates.find((candidate) => fs.existsSync(candidate)); +const withTrailingSlash = (baseUrl) => (baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`); + +/** + * Writes an llms.txt index (https://llmstxt.org) into the build output, plus a + * markdown twin of every docs page for it to link to. Both land at the docs + * root alongside sitemap.xml. See ./README.md. + */ +module.exports = function llmsTxtPlugin() { + return { + name: 'docusaurus-plugin-llms-txt', + + async postBuild(props) { + const { outDir, siteDir, i18n, plugins, siteConfig, baseUrl } = props; + + /** + * Each locale has its own outDir, so the Japanese build would write a + * `build/ja/llms.txt` that nothing links to, with section labels still + * taken from the English sidebar. + */ + if (i18n.currentLocale !== i18n.defaultLocale) { + return; + } + + const docsPlugin = plugins.find( + (plugin) => plugin.name === DOCS_PLUGIN_NAME && (plugin.options?.id ?? DOCS_PLUGIN_ID) === DOCS_PLUGIN_ID + ); + + if (!docsPlugin?.content?.loadedVersions) { + console.warn('[llms-txt] docs plugin content was not available, skipping llms.txt.'); + return; + } + + const { loadedVersions } = docsPlugin.content; + const staticDir = path.join(siteDir, 'static'); + const warn = (message) => console.warn(`[llms-txt] ${message}`); + + /** A missing or renamed source costs that page its snippets, not the build. */ + const readSource = (doc) => { + try { + return fs.readFileSync(path.join(siteDir, doc.source.replace(/^@site\//, '')), 'utf8'); + } catch { + warn(`could not read ${doc.source}, leaving its playground code out.`); + return ''; + } + }; + + // Twins cover every version, not just the current one. + const referencedByVersion = new Map( + loadedVersions.map((loaded) => [loaded.versionName, getReferencedDocIds(loaded.sidebars)]) + ); + + /** + * Every page that gets a twin, so a link between two of them can be + * rewritten to stay inside the markdown corpus. + */ + const twinUrls = new Map(); + for (const version of loadedVersions) { + for (const doc of version.docs) { + if (!doc.draft && !doc.unlisted && referencedByVersion.get(version.versionName).has(doc.id)) { + twinUrls.set(doc.permalink, `${withTrailingSlash(baseUrl)}${toMarkdownPath(doc.permalink, baseUrl)}`); + } + } + } + + let twins = 0; + for (const version of loadedVersions) { + const referencedIds = referencedByVersion.get(version.versionName); + + for (const doc of version.docs) { + // Matching what the index lists keeps scratch and redirected pages + // from being published as markdown nobody can reach. + if (doc.draft || doc.unlisted || !referencedIds.has(doc.id)) { + continue; + } + + const { htmlCandidates, markdownPath } = toOutputPaths(doc.permalink, { outDir, baseUrl }); + const htmlPath = firstExisting(htmlCandidates); + if (!htmlPath) { + warn(`no rendered HTML for ${doc.permalink}, skipping its markdown twin.`); + continue; + } + + const html = fs.readFileSync(htmlPath, 'utf8'); + const markdown = buildTwin({ + html, + pageUrl: `${siteConfig.url.replace(/\/+$/, '')}${doc.permalink}`, + usageDirs: hasPlaygrounds(html) ? readUsageDirs(readSource(doc)) : [], + staticDir, + twinUrls, + onWarn: warn, + }); + + if (markdown) { + fs.mkdirSync(path.dirname(markdownPath), { recursive: true }); + fs.writeFileSync(markdownPath, markdown); + twins += 1; + } + } + } + + const version = loadedVersions.find((loaded) => loaded.versionName === CURRENT_VERSION); + if (!version) { + warn(`no "${CURRENT_VERSION}" docs version found, skipping llms.txt.`); + return; + } + + const referencedIds = referencedByVersion.get(CURRENT_VERSION); + const docsById = new Map( + version.docs + .filter((doc) => !doc.draft && !doc.unlisted && referencedIds.has(doc.id)) + .map((doc) => [doc.id, doc]) + ); + + const { sections, optional } = buildSections({ sidebars: version.sidebars, docsById }); + + fs.writeFileSync( + path.join(outDir, 'llms.txt'), + renderLlmsTxt({ + title: siteConfig.title, + tagline: siteConfig.tagline, + intro: `${INTRO}${olderVersionsNote(loadedVersions)}`, + sections, + optional, + siteUrl: siteConfig.url, + baseUrl, + }) + ); + + const linkCount = sections.reduce((total, section) => total + section.docs.length, 0) + optional.length; + console.log(`[llms-txt] wrote ${twins} markdown twins and llms.txt with ${linkCount} links.`); + }, + }; +}; diff --git a/plugins/docusaurus-plugin-llms-txt/lib/llms-txt.js b/plugins/docusaurus-plugin-llms-txt/lib/llms-txt.js new file mode 100644 index 0000000000..218529d0d3 --- /dev/null +++ b/plugins/docusaurus-plugin-llms-txt/lib/llms-txt.js @@ -0,0 +1,180 @@ +const { toMarkdownUrl } = require('./paths'); + +/** + * Descriptions shorter than this are almost always a stray heading or a bare + * JSX tag rather than a sentence, so the bullet reads better without them. + */ +const MIN_DESCRIPTION_LENGTH = 15; + +/** Keeps a single bullet to roughly one terminal line. */ +const MAX_DESCRIPTION_LENGTH = 200; + +const REFERENCE_SIDEBARS = ['api', 'cli', 'native']; + +const normalizeWhitespace = (text) => + String(text ?? '') + .replace(/\s+/g, ' ') + .trim(); + +const forComparison = (text) => + normalizeWhitespace(text) + .toLowerCase() + .replace(/[^a-z0-9]/g, ''); + +/** + * Docusaurus falls back to a body excerpt when a page sets no frontmatter + * description, which is the usual case here. That excerpt is normally the + * page's SEO title and reads well, but it can land on a stray JSX tag or on a + * heading the title already says. Those are dropped so the bullet is + * title-only rather than misleading. + */ +function cleanDescription(description, title) { + const text = normalizeWhitespace(description); + + if (!text) { + return ''; + } + // Unclosed JSX or an import that the excerpt scraper did not strip. + if (/^[<{]/.test(text) || /^(import|export)\s/.test(text)) { + return ''; + } + if (text.length < MIN_DESCRIPTION_LENGTH) { + return ''; + } + // A description that just repeats the title adds nothing to the bullet. + // Both sides have to hold something first, or a description made purely of + // punctuation would match a missing title. + const comparableTitle = forComparison(title); + if (comparableTitle && forComparison(text) === comparableTitle) { + return ''; + } + + if (text.length <= MAX_DESCRIPTION_LENGTH) { + return text; + } + + const clipped = text.slice(0, MAX_DESCRIPTION_LENGTH); + const lastSpace = clipped.lastIndexOf(' '); + return `${(lastSpace > 0 ? clipped.slice(0, lastSpace) : clipped).replace(/[.,;:]$/, '')}...`; +} + +/** + * Visits the id of every doc a sidebar subtree points at, in sidebar order. + * + * Both `link` and `html` items are passed over. External links are not docs, + * and the internal ones, such as the "Responsive Grid" shortcut under Layout, + * point at a page that already appears under its own sidebar. + */ +function walkSidebarDocIds(items, visit) { + for (const item of items ?? []) { + if (item.type === 'doc' || item.type === 'ref') { + visit(item.id); + } else if (item.type === 'category') { + if (item.link?.type === 'doc') { + visit(item.link.id); + } + walkSidebarDocIds(item.items, visit); + } + } +} + +/** + * Every doc id reachable from any sidebar. Pages outside this set are + * unreachable from the site navigation, which is the signal used to skip them + * rather than a hand-kept path list. + */ +function getReferencedDocIds(sidebars) { + const ids = new Set(); + Object.values(sidebars ?? {}).forEach((items) => walkSidebarDocIds(items, (id) => ids.add(id))); + return ids; +} + +/** Flattens a sidebar subtree into the docs it points at, in sidebar order. */ +function collectDocs(items, docsById) { + const collected = []; + + walkSidebarDocIds(items, (id) => { + const doc = docsById.get(id); + if (doc) { + collected.push(doc); + } + }); + + return collected; +} + +/** + * Splits the sidebars into the named guide sections and the single `Optional` + * section of generated reference pages. + * + * A doc is listed once. The guide sidebar is walked first, so a page that also + * appears in a reference sidebar stays with its guide. + */ +function buildSections({ sidebars, docsById, referenceSidebars = REFERENCE_SIDEBARS }) { + const seen = new Set(); + + const unseen = (docs) => + docs.filter((doc) => { + if (seen.has(doc.id)) { + return false; + } + seen.add(doc.id); + return true; + }); + + const sections = (sidebars.docs ?? []) + .filter((item) => item.type === 'category') + .map((category) => ({ + title: category.label, + docs: unseen(collectDocs(category.items, docsById)), + })) + .filter((section) => section.docs.length > 0); + + const optional = unseen(referenceSidebars.flatMap((name) => collectDocs(sidebars[name] ?? [], docsById))); + + return { sections, optional }; +} + +const renderBullet = (doc, urlOptions) => { + const url = toMarkdownUrl(doc.permalink, urlOptions); + const description = cleanDescription(doc.description, doc.title); + return description ? `- [${doc.title}](${url}): ${description}` : `- [${doc.title}](${url})`; +}; + +const renderSection = (title, docs, urlOptions) => + [`## ${title}`, '', ...docs.map((doc) => renderBullet(doc, urlOptions)), ''].join('\n'); + +/** + * Renders the llms.txt body per the format at https://llmstxt.org: an H1, a + * blockquote summary, free prose, then H2-delimited link lists. `Optional` is + * the spec's reserved heading for links an agent can skip when it needs a + * shorter context. + */ +function renderLlmsTxt({ title, tagline, intro, sections, optional, siteUrl, baseUrl }) { + const urlOptions = { siteUrl, baseUrl }; + + const parts = [`# ${title}`, '']; + + if (tagline) { + parts.push(`> ${normalizeWhitespace(tagline)}`, ''); + } + if (intro) { + parts.push(normalizeWhitespace(intro), ''); + } + + sections.forEach((section) => parts.push(renderSection(section.title, section.docs, urlOptions))); + + if (optional.length > 0) { + parts.push(renderSection('Optional', optional, urlOptions)); + } + + return `${parts.join('\n').trimEnd()}\n`; +} + +module.exports = { + buildSections, + cleanDescription, + collectDocs, + getReferencedDocIds, + renderLlmsTxt, +}; diff --git a/plugins/docusaurus-plugin-llms-txt/lib/llms-txt.test.js b/plugins/docusaurus-plugin-llms-txt/lib/llms-txt.test.js new file mode 100644 index 0000000000..f2e7e85465 --- /dev/null +++ b/plugins/docusaurus-plugin-llms-txt/lib/llms-txt.test.js @@ -0,0 +1,217 @@ +import { describe, expect, it } from 'vitest'; + +import llmsTxt from './llms-txt.js'; + +const { buildSections, cleanDescription, collectDocs, getReferencedDocIds, renderLlmsTxt } = llmsTxt; + +const URL_OPTIONS = { siteUrl: 'https://ionicframework.com', baseUrl: '/docs/' }; + +const doc = (id, overrides = {}) => ({ + id, + title: id, + description: `A sentence about ${id}.`, + permalink: `/docs/${id}`, + ...overrides, +}); + +const toMap = (docs) => new Map(docs.map((entry) => [entry.id, entry])); + +describe('cleanDescription', () => { + it('keeps a normal sentence and collapses whitespace', () => { + expect(cleanDescription(' Routing and\nredirects in Angular apps. ', 'Angular Navigation')).toBe( + 'Routing and redirects in Angular apps.' + ); + }); + + it('drops JSX that leaked out of the excerpt scraper', () => { + expect(cleanDescription(' { + expect(cleanDescription('isPlatform', 'Platform')).toBe(''); + }); + + it('drops a description that only repeats the title', () => { + expect(cleanDescription('Customizing Animations', 'Customizing Animations')).toBe(''); + expect(cleanDescription('customizing animations!', 'Customizing Animations')).toBe(''); + }); + + it('keeps a short but genuine summary', () => { + expect(cleanDescription('Log in to Ionic', 'ionic login')).toBe('Log in to Ionic'); + }); + + it('truncates long text on a word boundary', () => { + const result = cleanDescription(`${'word '.repeat(80)}tail`, 'Some Page'); + + expect(result.endsWith('...')).toBe(true); + expect(result.length).toBeLessThanOrEqual(203); + expect(result).not.toContain('word w...'); + }); + + it('returns an empty string for missing input', () => { + expect(cleanDescription(undefined, 'Some Page')).toBe(''); + expect(cleanDescription('', 'Some Page')).toBe(''); + }); +}); + +describe('getReferencedDocIds', () => { + const sidebars = { + docs: [ + { + type: 'category', + label: 'Getting Started', + link: { type: 'doc', id: 'intro/overview' }, + items: [ + { type: 'doc', id: 'intro/cli' }, + { + type: 'category', + label: 'Nested', + items: [{ type: 'doc', id: 'intro/deep' }], + }, + { type: 'link', label: 'External', href: 'https://example.com' }, + { type: 'html', value: '
' }, + ], + }, + ], + api: [{ type: 'category', label: 'Button', items: [{ type: 'ref', id: 'api/button' }] }], + }; + + it('collects ids from every sidebar, including nested and category links', () => { + expect([...getReferencedDocIds(sidebars)].sort()).toEqual([ + 'api/button', + 'intro/cli', + 'intro/deep', + 'intro/overview', + ]); + }); + + it('leaves out a page that is present but reachable from no sidebar', () => { + const withOrphan = { + ...sidebars, + docs: [...sidebars.docs, { type: 'html', value: '

intro/first-app lives here but is not linked

' }], + }; + + expect(getReferencedDocIds(withOrphan).has('intro/first-app')).toBe(false); + }); + + it('handles missing sidebars without throwing', () => { + expect(getReferencedDocIds(undefined).size).toBe(0); + }); +}); + +describe('collectDocs', () => { + const docsById = toMap([doc('a'), doc('b'), doc('c')]); + + it('preserves sidebar order across nesting', () => { + const items = [ + { type: 'doc', id: 'a' }, + { type: 'category', label: 'Group', items: [{ type: 'doc', id: 'b' }] }, + { type: 'doc', id: 'c' }, + ]; + + expect(collectDocs(items, docsById).map((entry) => entry.id)).toEqual(['a', 'b', 'c']); + }); + + it('skips link and html items', () => { + const items = [ + { type: 'link', label: 'Responsive Grid', href: '/api/grid' }, + { type: 'html', value: '
' }, + { type: 'doc', id: 'a' }, + ]; + + expect(collectDocs(items, docsById).map((entry) => entry.id)).toEqual(['a']); + }); + + it('skips ids with no matching doc', () => { + const items = [ + { type: 'doc', id: 'a' }, + { type: 'doc', id: 'missing' }, + ]; + + expect(collectDocs(items, docsById).map((entry) => entry.id)).toEqual(['a']); + }); +}); + +describe('buildSections', () => { + const docsById = toMap([doc('intro/cli'), doc('theming/basics'), doc('api/button'), doc('cli/commands/build')]); + + const sidebars = { + docs: [ + { type: 'category', label: 'Getting Started', items: [{ type: 'doc', id: 'intro/cli' }] }, + { type: 'category', label: 'Theming', items: [{ type: 'doc', id: 'theming/basics' }] }, + { type: 'category', label: 'Empty', items: [{ type: 'doc', id: 'gone' }] }, + ], + api: [{ type: 'category', label: 'Button', items: [{ type: 'doc', id: 'api/button' }] }], + cli: [{ type: 'category', label: 'Commands', items: [{ type: 'doc', id: 'cli/commands/build' }] }], + native: [], + }; + + it('turns each top-level guide category into a section', () => { + const { sections } = buildSections({ sidebars, docsById }); + + expect(sections.map((section) => section.title)).toEqual(['Getting Started', 'Theming']); + expect(sections[0].docs.map((entry) => entry.id)).toEqual(['intro/cli']); + }); + + it('gathers the reference sidebars into the optional list', () => { + const { optional } = buildSections({ sidebars, docsById }); + + expect(optional.map((entry) => entry.id)).toEqual(['api/button', 'cli/commands/build']); + }); + + it('lists a doc once, keeping it with its guide section', () => { + const shared = { + docs: [{ type: 'category', label: 'Layout', items: [{ type: 'doc', id: 'api/button' }] }], + api: [{ type: 'category', label: 'Button', items: [{ type: 'doc', id: 'api/button' }] }], + }; + const { sections, optional } = buildSections({ sidebars: shared, docsById }); + + expect(sections[0].docs.map((entry) => entry.id)).toEqual(['api/button']); + expect(optional).toEqual([]); + }); +}); + +describe('renderLlmsTxt', () => { + const rendered = renderLlmsTxt({ + title: 'Ionic Framework', + tagline: 'The app platform for web developers', + intro: 'Every page below is linked as markdown.', + sections: [{ title: 'Getting Started', docs: [doc('intro/cli', { title: 'Ionic CLI' })] }], + optional: [doc('api/button', { title: 'ion-button', description: ' { + expect(rendered).toBe( + [ + '# Ionic Framework', + '', + '> The app platform for web developers', + '', + 'Every page below is linked as markdown.', + '', + '## Getting Started', + '', + '- [Ionic CLI](https://ionicframework.com/docs/intro/cli.md): A sentence about intro/cli.', + '', + '## Optional', + '', + '- [ion-button](https://ionicframework.com/docs/api/button.md)', + '', + ].join('\n') + ); + }); + + it('leaves out the Optional heading when there is nothing to put under it', () => { + const withoutOptional = renderLlmsTxt({ + title: 'Ionic Framework', + sections: [{ title: 'Getting Started', docs: [doc('intro/cli')] }], + optional: [], + ...URL_OPTIONS, + }); + + expect(withoutOptional).not.toContain('## Optional'); + }); +}); diff --git a/plugins/docusaurus-plugin-llms-txt/lib/markdown-twins.js b/plugins/docusaurus-plugin-llms-txt/lib/markdown-twins.js new file mode 100644 index 0000000000..38be7eb643 --- /dev/null +++ b/plugins/docusaurus-plugin-llms-txt/lib/markdown-twins.js @@ -0,0 +1,356 @@ +const path = require('path'); + +const { readPlaygroundCode, renderPlaygroundCode } = require('./playground-code'); + +/** + * Builds the markdown twin of a rendered page, repairing the HTML on its way + * into docusaurus-plugin-copy-page-button's converter. See ../README.md for + * what needs repairing and why. + * + * Deep-importing `src/htmlToMarkdown.js` reaches past the package's documented + * entry point. Its `files` list ships `src` and it declares no `exports` map, + * so the path resolves, but it is worth re-checking on upgrade. + */ +const { + convertToMarkdown, + extractPageMarkdownFromHtml, +} = require('docusaurus-plugin-copy-page-button/src/htmlToMarkdown.js'); + +if (typeof convertToMarkdown !== 'function' || typeof extractPageMarkdownFromHtml !== 'function') { + throw new Error( + 'docusaurus-plugin-llms-txt: docusaurus-plugin-copy-page-button/src/htmlToMarkdown.js no longer exports ' + + 'convertToMarkdown and extractPageMarkdownFromHtml. This deep import was written against 0.8.4.' + ); +} + +const PLAYGROUND_CONTAINER = /
]*>[\s\S]*?<\/span>/g; +const TAB_LIST_OR_PANEL = /]*role="?tablist"?[^>]*>([\s\S]*?)<\/ul>|
]*>/g; +const ADMONITION_HEADING = + /
]*>
]+)"?/g; +const ANCHOR_SENTINEL = '@@IONIC_ANCHOR@@'; +const ENCAPSULATION_PILL = /]*>[\s\S]*?<\/a>/g; +const FULLWIDTH_BAR = /\uFF5C/g; +const INTERNAL_LINK = /\]\((\/[^)\s#]+)(#[^)\s]*)?\)/g; + +const tableToken = (index) => `@@IONIC_TABLE_${index}@@`; +const playgroundToken = (index) => `@@IONIC_PLAYGROUND_${index}@@`; + +const cellToMarkdown = (html) => + convertToMarkdown(`
${html}
`).replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim(); + +/** + * Splits a table's inner HTML into rows of raw cell HTML. + * + * Written against the tag soup Docusaurus emits, with no closing `` or + * `` and a `` appearing mid-stream, rather than against + * well-formed markup. + */ +function parseRows(inner) { + return inner + .split(/]*>/) + .slice(1) + .map((row) => ({ + isHeader: /]/.test(row), + cells: row + .split(/]*>/) + .slice(1) + .map((cell) => cell.replace(/<\/t[hd]>|<\/tr>|<\/?tbody>|<\/?thead>|<\/table>/g, '')), + })) + .filter((row) => row.cells.length > 0); +} + +const toRow = (cells) => `| ${cells.join(' | ')} |`; + +/** + * Markdown has no way to express a table with no header row, and a fabricated + * header would read as data, so those become labeled lines instead. + */ +function renderTable(rows) { + const header = rows.find((row) => row.isHeader); + const body = rows.filter((row) => row !== header); + + if (!header) { + if (body.every((row) => row.cells.length === 2)) { + return body + .map(({ cells }) => `**${cells[0].replace(/^\*\*|\*\*$/g, '')}**: ${cells[1].replace(FULLWIDTH_BAR, '|')}`) + .join('\n\n'); + } + + const width = Math.max(...body.map((row) => row.cells.length)); + return [toRow(Array(width).fill('')), toRow(Array(width).fill('---')), ...body.map((row) => toRow(row.cells))].join( + '\n' + ); + } + + return [toRow(header.cells), toRow(header.cells.map(() => '---')), ...body.map((row) => toRow(row.cells))].join('\n'); +} + +function replaceTables(html) { + const tables = []; + + const withTokens = html.replace(/]*>([\s\S]*?)<\/table>/g, (match, inner) => { + const rows = parseRows(inner); + if (rows.length === 0) { + return match; + } + + // A header with no body rows is a client-rendered table. Emitting it + // reads as an authoritative "there is nothing here". + if (rows.every((row) => row.isHeader)) { + return ''; + } + + tables.push(renderTable(rows.map(({ isHeader, cells }) => ({ isHeader, cells: cells.map(cellToMarkdown) })))); + return `

${tableToken(tables.length - 1)}

`; + }); + + return { html: withTokens, tables }; +} + +/** Whether a page renders any playground, so its source is only read if needed. */ +function hasPlaygrounds(html) { + return html.includes('playground__container'); +} + +/** + * Marks each playground with a placeholder. + * + * The marker goes immediately before the container rather than replacing it, + * which keeps this clear of having to match the container's closing tag + * through its nested markup. The shell itself still converts to nothing. + */ +function markPlaygrounds(html) { + let count = 0; + + const withTokens = html.replace(PLAYGROUND_CONTAINER, (match) => { + count += 1; + return `

${playgroundToken(count - 1)}

${match}`; + }); + + return { html: withTokens, count }; +} + +/** + * Usage folders a page renders, in the order its playgrounds appear. + * + * Pairing is positional, so this has to follow render order rather than + * import order. `api/datetime.mdx` groups its imports by feature at the top + * and then renders the sections in reading order, so the two sequences differ + * and pairing on imports hands a section another section's code. A count + * check cannot catch that, since a permutation has the same length. + */ +function readUsageDirs(source) { + const byName = new Map( + [...source.matchAll(/import\s+(\w+)\s+from\s+'@site\/static\/usage\/([\w\-/.]+)\/index\.mdx'/g)].map((match) => [ + match[1], + match[2], + ]) + ); + + if (byName.size === 0) { + return []; + } + + // Match a self-closing tag carrying props as well as a bare one. + const body = source.replace(/^import\s[^\n]*$/gm, ''); + return [...body.matchAll(/<([A-Z]\w*)[\s/>]/g)].map((match) => byName.get(match[1])).filter(Boolean); +} + +const textOf = (html) => + html + .replace(/<[^>]*>/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + +/** + * Moves each tab's label into its panel. + * + * The labels live in the tab strip, which is chrome, but the panels are + * mutually exclusive alternatives. Dropping the strip on its own leaves them + * concatenated and unattributed, which on the framework tabs means near + * identical prose with contradicting imports. + * + * Pairing is positional, so a group is only labeled when its counts line up. + * Docusaurus emits one tab per `values` entry but one panel per `TabItem`, + * and a page can declare fewer values than it renders items, which would + * otherwise shift every later label onto its neighbor's content. Scanning + * before rewriting is what makes that check possible, since a single pass + * cannot take back a label it has already written. + */ +function labelTabs(html, onWarn = () => {}) { + const scan = new RegExp(TAB_LIST_OR_PANEL.source, 'g'); + const groups = []; + let group = null; + + for (let match = scan.exec(html); match; match = scan.exec(html)) { + if (match[1] !== undefined) { + if (group) { + groups.push(group); + } + group = { + labels: [...match[1].matchAll(/]*>([\s\S]*?)(?= textOf(item[1])), + panels: [], + }; + } else if (group) { + group.panels.push(match.index); + } + } + if (group) { + groups.push(group); + } + + const labelAt = new Map(); + for (const { labels, panels } of groups) { + if (labels.length !== panels.length) { + onWarn(`tab group has ${labels.length} labels for ${panels.length} panels, leaving it unlabeled.`); + continue; + } + panels.forEach((offset, index) => labelAt.set(offset, labels[index])); + } + + return html.replace(TAB_LIST_OR_PANEL, (whole, list, offset) => { + if (list !== undefined) { + return ''; + } + const label = labelAt.get(offset); + return label ? `${whole}

${label}

` : whole; + }); +} + +/** + * The converter looks for a bare `admonition` class, which Docusaurus no + * longer emits, so the heading falls through as the bare word "info". This + * turns it into a labeled line and keeps any custom title with its type. + */ +const labelAdmonitions = (html) => + html.replace(ADMONITION_HEADING, (whole, type, heading) => { + const title = textOf(heading); + const name = type.charAt(0).toUpperCase() + type.slice(1); + const label = title && title.toLowerCase() !== type ? `${name}: ${title}` : name; + return `${whole.slice(0, whole.indexOf('>${label}

`; + }); + +/** + * Applies `transform` to each line that is not inside a fenced code block. + * + * Fences are tracked by their own marker run, so a ```` block + * wrapping a ``` sample does not flip the parity, and a fence indented inside + * a list item still counts as code. + */ +function mapProseLines(markdown, transform) { + let fence = null; + + return markdown + .split('\n') + .map((line) => { + const match = /^(\s*)(`{3,}|~{3,})(.*)$/.exec(line); + + if (fence) { + if (match && match[2][0] === fence.marker && match[2].length >= fence.length && !match[3].trim()) { + fence = null; + } + return line; + } + if (match) { + fence = { marker: match[2][0], length: match[2].length }; + return line; + } + return transform(line); + }) + .join('\n'); +} + +/** + * The converter puts a space after every inline `code`, link and bold run, + * which leaves a gap before the punctuation that follows. + */ +const closePunctuationGaps = (markdown) => + mapProseLines(markdown, (line) => + line.replace(/([`)*\w]) +([,.;:!?)\]])(?=[\s*`)\]]|$)/g, '$1$2').replace(/([([]) +(?=[`*\w])/g, '$1') + ); + +const tidy = (markdown) => closePunctuationGaps(markdown.replace(/\n{3,}/g, '\n\n')).trim(); + +/** + * @param {object} args + * @param {string} args.html rendered page HTML + * @param {string} args.pageUrl canonical URL of the page + * @param {string[]} args.usageDirs usage folders, page order, relative to static/usage + * @param {string} args.staticDir absolute path of the site's static directory + * @param {Map} [args.twinUrls] permalink to twin path, for cross-links + * @param {(message: string) => void} [args.onWarn] + */ +function buildTwin({ html, pageUrl, usageDirs = [], staticDir, twinUrls = new Map(), onWarn = () => {} }) { + /** + * The version badge and the tab strip are chrome. The encapsulation pill is + * worth keeping but sits loose between the header and the first paragraph, + * so without a block of its own it runs into the opening sentence. + */ + const cleaned = labelAdmonitions(labelTabs(html.replace(VERSION_BADGE, ''), onWarn)) + .replace(ENCAPSULATION_PILL, (pill) => `

Encapsulation: ${pill}

`) + // The converter drops href="#x", treating it as a heading affordance. On + // a self-contained page those are real cross-references, so they are + // carried past it behind a sentinel. + .replace(ANCHOR_HREF, `href="${ANCHOR_SENTINEL}#$1"`); + + const tabled = replaceTables(cleaned); + const marked = markPlaygrounds(tabled.html); + + // A count that does not line up means the page order assumption no longer + // holds, so leave the snippets out rather than attach them to the wrong one. + const spliceCode = marked.count === usageDirs.length; + if (!spliceCode && marked.count > 0) { + onWarn(`${pageUrl}: found ${marked.count} playgrounds for ${usageDirs.length} usage imports, leaving code out.`); + } + + let markdown = extractPageMarkdownFromHtml(marked.html, pageUrl, { requireDocContent: true }); + if (!markdown.trim()) { + return ''; + } + + tabled.tables.forEach((table, index) => { + markdown = markdown.replace(new RegExp(`[ \\t]*${tableToken(index)}`), () => `\n\n${table}\n\n`); + }); + + for (let index = 0; index < marked.count; index += 1) { + const code = spliceCode + ? renderPlaygroundCode(readPlaygroundCode(path.join(staticDir, 'usage', usageDirs[index]))) + : ''; + markdown = markdown.replace(playgroundToken(index), () => code); + } + + /** + * Leave a link alone unless its target has a twin, so anchors, assets and + * pages that were skipped keep pointing at something that exists. + */ + const linked = mapProseLines(tidy(markdown), (line) => + line.replace(INTERNAL_LINK, (whole, target, hash) => { + // Some sources link a page by its source filename. Those resolve to the + // same twin, so the extension is dropped before looking the target up. + const withoutExtension = target.replace(/\.mdx?$/, ''); + const twin = twinUrls.get(target) ?? twinUrls.get(withoutExtension); + return twin ? `](${twin}${hash ?? ''})` : whole; + }) + ) + .replaceAll(`(${ANCHOR_SENTINEL}#`, '(#') + // Raw anchor HTML that the converter passed through as text keeps its + // sentinel, since it never became a markdown link. + .replaceAll(ANCHOR_SENTINEL, ''); + + return `${linked}\n`; +} + +module.exports = { + buildTwin, + closePunctuationGaps, + labelTabs, + mapProseLines, + hasPlaygrounds, + markPlaygrounds, + parseRows, + readUsageDirs, + renderTable, + replaceTables, +}; diff --git a/plugins/docusaurus-plugin-llms-txt/lib/markdown-twins.test.js b/plugins/docusaurus-plugin-llms-txt/lib/markdown-twins.test.js new file mode 100644 index 0000000000..9742843ffb --- /dev/null +++ b/plugins/docusaurus-plugin-llms-txt/lib/markdown-twins.test.js @@ -0,0 +1,395 @@ +import { describe, expect, it } from 'vitest'; + +import twins from './markdown-twins.js'; + +const { + buildTwin, + closePunctuationGaps, + labelTabs, + mapProseLines, + markPlaygrounds, + parseRows, + readUsageDirs, + renderTable, + replaceTables, +} = twins; + +/** The shape Docusaurus emits: minified, with the optional end tags left out. */ +const HEADERLESS_TABLE = + '
' + + '
DescriptionThe type of button.' + + '
Attributebutton-type' + + '
Typestring' + + '
'; + +const HEADER_TABLE = + '' + + '
ConfigType
alertEnterAnimationBuilder' + + '
alertLeaveAnimationBuilder' + + '
'; + +const page = (body) => `
${body}
`; + +describe('parseRows', () => { + it('separates rows even though the end tags are missing', () => { + const rows = parseRows(HEADERLESS_TABLE.replace(/^.*?|<\/table>.*$/g, '')); + + expect(rows).toHaveLength(3); + expect(rows.every((row) => row.cells.length === 2)).toBe(true); + expect(rows[0].isHeader).toBe(false); + }); + + it('marks the header row', () => { + const rows = parseRows(HEADER_TABLE.replace(/^
|<\/table>$/g, '')); + + expect(rows.map((row) => row.isHeader)).toEqual([true, false, false]); + expect(rows[0].cells).toEqual(['Config', 'Type']); + }); +}); + +describe('renderTable', () => { + it('turns a headerless pair table into labeled lines', () => { + const rendered = renderTable([ + { isHeader: false, cells: ['**Description**', 'The type of button.'] }, + { isHeader: false, cells: ['**Type**', '`string`'] }, + ]); + + expect(rendered).toBe('**Description**: The type of button.\n\n**Type**: `string`'); + }); + + it('restores a real pipe in a type union, since there is no table to escape for', () => { + const rendered = renderTable([{ isHeader: false, cells: ['**Type**', '`"ios" \uFF5C "md"`'] }]); + + expect(rendered).toBe('**Type**: `"ios" | "md"`'); + }); + + it('gives a header table the separator row markdown needs', () => { + const rendered = renderTable([ + { isHeader: true, cells: ['Config', 'Type'] }, + { isHeader: false, cells: ['`alertEnter`', '`AnimationBuilder`'] }, + ]); + + expect(rendered).toBe('| Config | Type |\n| --- | --- |\n| `alertEnter` | `AnimationBuilder` |'); + }); + + it('falls back to a blank header when a headerless table is not a pair table', () => { + const rendered = renderTable([{ isHeader: false, cells: ['a', 'b', 'c'] }]); + + expect(rendered).toBe('| | | |\n| --- | --- | --- |\n| a | b | c |'); + }); +}); + +describe('replaceTables', () => { + it('swaps each table for a placeholder and converts its cells', () => { + const { html, tables } = replaceTables(HEADERLESS_TABLE); + + expect(html).toContain('@@IONIC_TABLE_0@@'); + expect(html).not.toContain('
'); + expect(tables[0]).toBe( + '**Description**: The type of button.\n\n**Attribute**: `button-type`\n\n**Type**: `string`' + ); + }); + + it('escapes a pipe inside a cell so it cannot break the row', () => { + const { tables } = replaceTables('
A
x | y
'); + + expect(tables[0]).toContain('x \\| y'); + }); +}); + +describe('markPlaygrounds', () => { + it('marks each container without having to match its closing tag', () => { + const { html, count } = markPlaygrounds( + '
a
b
' + ); + + expect(count).toBe(2); + expect(html.indexOf('@@IONIC_PLAYGROUND_0@@')).toBeLessThan(html.indexOf('@@IONIC_PLAYGROUND_1@@')); + }); +}); + +describe('readUsageDirs', () => { + it('reads usage folders in render order, including camelCase paths', () => { + const source = [ + "import Trigger from '@site/static/usage/v9/alert/presenting/trigger/index.mdx';", + "import IsOpen from '@site/static/usage/v9/alert/presenting/isOpen/index.mdx';", + "import Other from '@theme/Tabs';", + '', + '', + '', + ].join('\n'); + + expect(readUsageDirs(source)).toEqual(['v9/alert/presenting/trigger', 'v9/alert/presenting/isOpen']); + }); + + it('follows the order the playgrounds render, not the order they are imported', () => { + const source = [ + "import Later from '@site/static/usage/v9/datetime/format-options/index.mdx';", + "import Earlier from '@site/static/usage/v9/datetime/localization/custom-locale/index.mdx';", + '', + '', + '', + ].join('\n'); + + expect(readUsageDirs(source)).toEqual(['v9/datetime/localization/custom-locale', 'v9/datetime/format-options']); + }); + + it('matches a self-closing tag that carries props', () => { + const source = [ + "import Nav from '@site/static/usage/v9/react/navigation/index.mdx';", + '', + '