diff --git a/src/__tests__/__snapshots__/collection.patternFlyApi.test.ts.snap b/src/__tests__/__snapshots__/collection.patternFlyApi.test.ts.snap
index ed8d6a68..1777a931 100644
--- a/src/__tests__/__snapshots__/collection.patternFlyApi.test.ts.snap
+++ b/src/__tests__/__snapshots__/collection.patternFlyApi.test.ts.snap
@@ -5,19 +5,23 @@ exports[`collectionCallback should match snapshot for collection result 1`] = `
"records": [
{
"data": {
- "card": {
- "category": "css",
- "description": "Card css content",
- "displayName": "Card",
- "id": "api::v1::components::card::css::0",
- "path": "https://main.patternfly-org.pages.dev/api/v1/components/Card/css",
- "pathSlug": "card",
- "section": "components",
- "source": "api",
- "version": "v1",
- },
+ "card": [
+ {
+ "category": "css",
+ "content": "Card css content",
+ "contentType": "",
+ "description": "PatternFly variables and tokens for Card CSS.",
+ "displayName": "Card CSS",
+ "id": "api::v1::components::card::css",
+ "path": "https://main.patternfly-org.pages.dev/api/v1/components/Card/css",
+ "pathSlug": "components-card-css",
+ "section": "components",
+ "source": "api",
+ "version": "v1",
+ },
+ ],
},
- "id": "api::v1::components::card::css::0",
+ "id": "api::v1::components::card::css",
"sourceId": "https://main.patternfly-org.pages.dev/api/v1/components/Card/css",
"sourceType": "api",
},
diff --git a/src/__tests__/__snapshots__/options.defaults.test.ts.snap b/src/__tests__/__snapshots__/options.defaults.test.ts.snap
index 9aff8168..1c705c69 100644
--- a/src/__tests__/__snapshots__/options.defaults.test.ts.snap
+++ b/src/__tests__/__snapshots__/options.defaults.test.ts.snap
@@ -57,9 +57,16 @@ exports[`options defaults should return specific properties: defaults 1`] = `
"props",
"css",
],
- "crawlCancelMs": 180000,
- "crawlIntervalMs": 43200000,
"enabled": false,
+ "schedule": {
+ "continueOnError": true,
+ "intervalMs": 604800000,
+ "repeat": Infinity,
+ },
+ "timeoutMs": 120000,
+ "traversalPaths": [
+ "examples",
+ ],
"versions": "https://main.patternfly-org.pages.dev/api/versions",
},
"availableResourceVersions": [
diff --git a/src/__tests__/collection.patternFlyApi.test.ts b/src/__tests__/collection.patternFlyApi.test.ts
index f1386894..b4683ff0 100644
--- a/src/__tests__/collection.patternFlyApi.test.ts
+++ b/src/__tests__/collection.patternFlyApi.test.ts
@@ -7,6 +7,7 @@ import {
crawler
} from '../collection.patternFlyApi';
import { processDocsFunction } from '../server.getResources';
+import { getOptions } from '../options.context';
jest.mock('../server.getResources');
@@ -46,12 +47,11 @@ describe('collectionCallback', () => {
isSuccess: true
}
])
- // crawler to leaf at a component facet path ("props")
.mockResolvedValueOnce([
{
- content: 'Button props content',
+ content: 'Button react component content with length enough to pass quality scoring...',
path: `${BASE}/v1/components/Button`,
- resolvedPath: `${BASE}/v1/components/Button/props`,
+ resolvedPath: `${BASE}/v1/components/Button/react`,
isSuccess: true
}
]);
@@ -76,22 +76,26 @@ describe('collectionCallback', () => {
expect(keys.length).toBe(1);
const key: any = keys[0];
+ expect(key).toBe('button');
+
expect(first).toMatchObject({
- sourceId: `${BASE}/v1/components/Button/props`
+ sourceId: `${BASE}/v1/components/Button/react`
});
- expect(first.data[key]).toMatchObject({
+ expect(Array.isArray(first.data[key])).toBe(true);
+
+ expect(first.data[key][0]).toMatchObject({
displayName: 'Button',
- pathSlug: 'button',
+ pathSlug: 'components-button-react',
source: 'api',
version: 'v1',
section: 'components',
- category: 'props',
- path: `${BASE}/v1/components/Button/props`
+ category: 'react',
+ path: `${BASE}/v1/components/Button/react`
});
});
- it('should uses kind "doc" when a facet is not a componentPath', async () => {
+ it('should use an extrapolated category', async () => {
// getVersions to ["v1"]
mockedProcessDocsFunction
.mockResolvedValueOnce([
@@ -118,15 +122,15 @@ describe('collectionCallback', () => {
const rec: any = result.records[0];
// id encodes version, section, item, kind, and index
- expect(rec?.id).toMatch(/^api::v1::components::card::doc::0$/);
+ expect(rec?.id).toMatch(/^api::v1::components::card::overview$/);
const key: any = rec?.data ? Object.keys(rec.data)[0] : '';
expect(key).toBe('card');
- expect(rec?.data?.[key]).toMatchObject({
+ expect(rec?.data?.[key]).toContainEqual(expect.objectContaining({
displayName: 'Card',
- category: 'doc'
- });
+ category: 'overview'
+ }));
});
it('should match snapshot for collection result', async () => {
@@ -205,7 +209,7 @@ describe('crawler', () => {
expect(res).toHaveLength(1);
expect(res[0]?.content).toBe('some content');
- expect(mockedProcessDocsFunction).toHaveBeenCalledTimes(2);
+ expect(mockedProcessDocsFunction).toHaveBeenCalledTimes(3);
});
it('handles component paths and terminates recursion', async () => {
@@ -266,6 +270,16 @@ describe('crawler', () => {
expect(res.length).toBeGreaterThanOrEqual(1);
expect(mockedProcessDocsFunction).toHaveBeenCalledWith(['https://api.com/v1']);
});
+
+ it('aborts crawling early when signal is aborted', async () => {
+ const controller = new AbortController();
+
+ controller.abort();
+ const res = await crawler(['https://api.com/v1'], { signal: controller.signal });
+
+ expect(res).toEqual([]);
+ expect(mockedProcessDocsFunction).not.toHaveBeenCalled();
+ });
});
describe('apiSpider', () => {
@@ -288,7 +302,7 @@ describe('apiSpider', () => {
expect(res).toEqual([]);
});
- it('returns ApiContent[] with metadata shape', async () => {
+ it('returns ApiContent[] shape', async () => {
mockedProcessDocsFunction
.mockResolvedValueOnce([
{
@@ -311,14 +325,37 @@ describe('apiSpider', () => {
expect(res.length).toBeGreaterThan(0);
expect(res[0]).toMatchObject({
- url: 'https://main.patternfly-org.pages.dev/api/v1/section/item/facet',
- content: 'leaf content',
- semanticContext: {
- version: 'v1',
- section: 'section',
- item: 'item',
- facet: 'facet'
+ path: 'https://main.patternfly-org.pages.dev/api/v1',
+ resolvedPath: 'https://main.patternfly-org.pages.dev/api/v1/section/item/facet',
+ content: 'leaf content'
+ });
+ });
+
+ it('handles crawl timeout gracefully in apiSpider', async () => {
+ const options = getOptions();
+
+ mockedProcessDocsFunction
+ .mockResolvedValueOnce([
+ {
+ content: JSON.stringify(['v1']),
+ path: 'https://main.patternfly-org.pages.dev/api/versions',
+ resolvedPath: 'https://main.patternfly-org.pages.dev/api/versions',
+ isSuccess: true
+ }
+ ])
+ .mockImplementation(() => new Promise(() => {}));
+
+ const res = await apiSpider({
+ ...options,
+ patternflyOptions: {
+ ...options.patternflyOptions,
+ api: {
+ ...options.patternflyOptions.api,
+ timeoutMs: 20
+ }
}
});
+
+ expect(res).toEqual([]);
});
});
diff --git a/src/__tests__/collection.patternFlyApiHelpers.test.ts b/src/__tests__/collection.patternFlyApiHelpers.test.ts
new file mode 100644
index 00000000..0a6ae48f
--- /dev/null
+++ b/src/__tests__/collection.patternFlyApiHelpers.test.ts
@@ -0,0 +1,822 @@
+import {
+ calculateContentQualityScore,
+ extractApiDescription,
+ extractApiDisplayName,
+ extractApiName,
+ formatSlugToTitle,
+ getApiFallbackDescription,
+ getLiveExampleCount,
+ hasEmptyFileCodeFence,
+ hasLiveExample,
+ isRawImport,
+ normalizeSlug
+} from '../collection.patternFlyApiHelpers';
+
+describe('isRawImport', () => {
+ it.each([
+ {
+ description: 'default import with ?raw query',
+ input: "import Button from './Button.tsx?raw'",
+ expected: true
+ },
+ {
+ description: 'named import with ?raw query',
+ input: "import { Button } from './Button.tsx?raw'",
+ expected: true
+ },
+ {
+ description: 'multiple named imports with ?raw query',
+ input: "import { Button, Card, Modal } from './components?raw'",
+ expected: true
+ },
+ {
+ description: 'namespace import with ?raw query',
+ input: "import * as React from './React?raw'",
+ expected: true
+ },
+ {
+ description: 'combined default and named imports with ?raw query',
+ input: "import React, { useState } from './React?raw'",
+ expected: true
+ },
+ {
+ description: 'double quotes with ?raw query',
+ input: 'import Button from "./Button.tsx?raw"',
+ expected: true
+ },
+ {
+ description: 'case-insensitive import statement with uppercase ?RAW',
+ input: "IMPORT Button FROM './Button.tsx?RAW'",
+ expected: true
+ },
+ {
+ description: 'multiline import statement with ?raw query',
+ input: "import {\n Button,\n Card\n} from './components?raw'",
+ expected: true
+ },
+ {
+ description: 'standard import without ?raw query',
+ input: "import Button from './Button.tsx'",
+ expected: false
+ },
+ {
+ description: 'side-effect import without clause',
+ input: "import './styles.css?raw'",
+ expected: false
+ },
+ {
+ description: 'string without import statement',
+ input: 'const file = "./Button.tsx?raw";',
+ expected: false
+ },
+ {
+ description: 'empty string',
+ input: '',
+ expected: false
+ }
+ ])('should detect raw imports, $description', ({ input, expected }) => {
+ expect(isRawImport(input)).toBe(expected);
+ });
+});
+
+describe('hasLiveExample', () => {
+ it.each([
+ {
+ description: 'self-closing LiveExample tag',
+ input: '',
+ expected: true
+ },
+ {
+ description: 'opening LiveExample tag with attributes',
+ input: '',
+ expected: true
+ },
+ {
+ description: 'case-insensitive liveexample tag',
+ input: '',
+ expected: true
+ },
+ {
+ description: 'LiveExample tag with multiline attributes',
+ input: '',
+ expected: true
+ },
+ {
+ description: 'LiveExample tag without attributes',
+ input: '',
+ expected: true
+ },
+ {
+ description: 'text without LiveExample tag',
+ input: 'Regular HTML component
',
+ expected: false
+ },
+ {
+ description: 'extended component name without word boundary match',
+ input: '',
+ expected: false
+ },
+ {
+ description: 'empty string',
+ input: '',
+ expected: false
+ }
+ ])('should detect LiveExample tags, $description', ({ input, expected }) => {
+ expect(hasLiveExample(input)).toBe(expected);
+ });
+});
+
+describe('getLiveExampleCount', () => {
+ it.each([
+ {
+ description: 'zero occurrences in plain text',
+ input: 'Plain text without examples',
+ expected: 0
+ },
+ {
+ description: 'single self-closing tag',
+ input: '',
+ expected: 1
+ },
+ {
+ description: 'multiple tags with mixed casing',
+ input: '\n\n',
+ expected: 3
+ },
+ {
+ description: 'tag with child elements',
+ input: 'child content',
+ expected: 1
+ },
+ {
+ description: 'empty string',
+ input: '',
+ expected: 0
+ }
+ ])('should count LiveExample occurrences, $description', ({ input, expected }) => {
+ expect(getLiveExampleCount(input)).toBe(expected);
+ });
+});
+
+describe('hasEmptyFileCodeFence', () => {
+ it.each([
+ {
+ description: 'empty code fence with file attribute',
+ input: '```ts file="./ButtonBasic.tsx"\n```',
+ expected: true
+ },
+ {
+ description: 'empty code fence with file attribute and inner whitespace',
+ input: '```tsx file="./Button.tsx" \n ```',
+ expected: true
+ },
+ {
+ description: 'empty code fence with language only',
+ input: '```ts\n```',
+ expected: true
+ },
+ {
+ description: 'empty code fence with no language',
+ input: '```\n```',
+ expected: true
+ },
+ {
+ description: 'empty code fence with trailing whitespace inside',
+ input: '```js\n \n```',
+ expected: true
+ },
+ {
+ description: 'non-empty code fence with file attribute',
+ input: '```ts file="./Button.tsx"\nconst button = true;\n```',
+ expected: false
+ },
+ {
+ description: 'non-empty code fence with language',
+ input: '```ts\nconst total = 42;\n```',
+ expected: false
+ },
+ {
+ description: 'plain text without code fences',
+ input: 'Just regular documentation text',
+ expected: false
+ },
+ {
+ description: 'empty string',
+ input: '',
+ expected: false
+ }
+ ])('should detect empty file code fences, $description', ({ input, expected }) => {
+ expect(hasEmptyFileCodeFence(input)).toBe(expected);
+ });
+});
+
+describe('calculateContentQualityScore', () => {
+ it.each([
+ {
+ description: 'undefined content returns baseScore',
+ content: undefined,
+ options: undefined,
+ expected: 1
+ },
+ {
+ description: 'null content returns baseScore',
+ content: null,
+ options: undefined,
+ expected: 1
+ },
+ {
+ description: 'boolean content returns baseScore',
+ content: true,
+ options: undefined,
+ expected: 1
+ },
+ {
+ description: 'empty string returns baseScore',
+ content: '',
+ options: undefined,
+ expected: 1
+ },
+ {
+ description: 'whitespace-only string returns baseScore',
+ content: ' \n\t ',
+ options: undefined,
+ expected: 1
+ },
+ {
+ description: 'category equals examples skips quality checks',
+ content: 'short',
+ options: { category: 'examples' },
+ expected: 1
+ },
+ {
+ description: 'high quality content exceeding minimum characters',
+ content: 'A'.repeat(200),
+ options: undefined,
+ expected: 1
+ },
+ {
+ description: 'short content without code block receives length penalty',
+ content: 'Short content description.',
+ options: undefined,
+ expected: 0.97
+ },
+ {
+ description: 'short content with code block does not receive length penalty',
+ content: 'Short:\n```ts\nconst value = 1;\n```',
+ options: undefined,
+ expected: 1
+ },
+ {
+ description: 'invalid JSON-like content receives JSON penalty and length penalty',
+ content: '{ invalid: json content }',
+ options: undefined,
+ expected: 0.94
+ },
+ {
+ description: 'valid JSON content without length penalty when over minimum length',
+ content: JSON.stringify({ description: 'A'.repeat(160) }),
+ options: undefined,
+ expected: 1
+ },
+ {
+ description: 'content with raw import receives penalty',
+ content: `import Button from './Button?raw';\n${'A'.repeat(160)}`,
+ options: undefined,
+ expected: 0.97
+ },
+ {
+ description: 'content with multiple LiveExample tags reduces score per tag',
+ content: `\n\n${'A'.repeat(160)}`,
+ options: undefined,
+ expected: 0.94
+ },
+ {
+ description: 'content with empty file code fence over minimum characters',
+ content: `${'A'.repeat(160)}\n\`\`\`ts file="./Button.tsx"\n\`\`\``,
+ options: undefined,
+ expected: 0.97
+ },
+ {
+ description: 'content with empty file code fence under minimum characters receives double penalty',
+ content: '```ts file="./Button.tsx"\n```',
+ options: undefined,
+ expected: 0.94
+ },
+ {
+ description: 'numeric content converted to string and evaluated',
+ content: 12345,
+ options: undefined,
+ expected: 0.97
+ },
+ {
+ description: 'custom baseScore and qualityReduction options',
+ content: 'Short text',
+ options: { baseScore: 0.8, qualityReduction: 0.1 },
+ expected: 0.7
+ },
+ {
+ description: 'custom minCharacters option satisfied',
+ content: 'A'.repeat(60),
+ options: { minCharacters: 50 },
+ expected: 1
+ },
+ {
+ description: 'score clamped to 0 when penalties exceed baseScore',
+ content: '{ invalid json }',
+ options: { baseScore: 0.05, qualityReduction: 0.1 },
+ expected: 0
+ },
+ {
+ description: 'score clamped to 1 when baseScore exceeds 1',
+ content: 'A'.repeat(200),
+ options: { baseScore: 1.5 },
+ expected: 1
+ }
+ ])('should calculate quality score, $description', ({ content, options, expected }: any) => {
+ expect(calculateContentQualityScore(content, options)).toBe(expected);
+ });
+});
+
+describe('normalizeSlug', () => {
+ it.each([
+ {
+ description: 'PascalCase component name',
+ input: 'Button',
+ expected: 'button'
+ },
+ {
+ description: 'multi-word PascalCase component name',
+ input: 'ActionList',
+ expected: 'action-list'
+ },
+ {
+ description: 'multi-word PascalCase with three words',
+ input: 'ModalBoxHeader',
+ expected: 'modal-box-header'
+ },
+ {
+ description: 'acronym exception CSS',
+ input: 'CSS',
+ expected: 'css'
+ },
+ {
+ description: 'acronym exception HTML',
+ input: 'HTML',
+ expected: 'html'
+ },
+ {
+ description: 'acronym exception AI',
+ input: 'AI',
+ expected: 'ai'
+ },
+ {
+ description: 'acronym exception MCP',
+ input: 'MCP',
+ expected: 'mcp'
+ },
+ {
+ description: 'acronym exception CLI',
+ input: 'CLI',
+ expected: 'cli'
+ },
+ {
+ description: 'acronym exception UXD',
+ input: 'UXD',
+ expected: 'uxd'
+ },
+ {
+ description: 'acronym exception UI',
+ input: 'UI',
+ expected: 'ui'
+ },
+ {
+ description: 'acronym exception API',
+ input: 'API',
+ expected: 'api'
+ },
+ {
+ description: 'acronym exception FAQ and FAQS',
+ input: 'FAQS',
+ expected: 'faqs'
+ },
+ {
+ description: 'acronym exception ARIA',
+ input: 'ARIA',
+ expected: 'aria'
+ },
+ {
+ description: 'acronym exception RTL',
+ input: 'RTL',
+ expected: 'rtl'
+ },
+ {
+ description: 'already kebab-cased string',
+ input: 'action-list',
+ expected: 'action-list'
+ },
+ {
+ description: 'snake_cased string',
+ input: 'action_list_item',
+ expected: 'action-list-item'
+ },
+ {
+ description: 'mixed underscores and multiple hyphens',
+ input: 'action__list--item_sub',
+ expected: 'action-list-item-sub'
+ },
+ {
+ description: 'leading and trailing whitespace',
+ input: ' Button ',
+ expected: 'button'
+ },
+ {
+ description: 'camelCase string with leading lowercase',
+ input: 'actionList',
+ expected: 'actionlist'
+ },
+ {
+ description: 'empty string',
+ input: '',
+ expected: ''
+ }
+ ])('should normalize slug, $description', ({ input, expected }) => {
+ expect(normalizeSlug(input)).toBe(expected);
+ });
+});
+
+describe('formatSlugToTitle', () => {
+ it.each([
+ {
+ description: 'empty slug returns default title',
+ slug: '',
+ section: undefined,
+ expected: 'PatternFly API'
+ },
+ {
+ description: 'single word slug',
+ slug: 'button',
+ section: undefined,
+ expected: 'Button'
+ },
+ {
+ description: 'kebab-case slug',
+ slug: 'action-list',
+ section: undefined,
+ expected: 'Action List'
+ },
+ {
+ description: 'slug with acronyms',
+ slug: 'css-variables',
+ section: undefined,
+ expected: 'CSS Variables'
+ },
+ {
+ description: 'slug with multiple acronyms',
+ slug: 'html-and-css-api',
+ section: undefined,
+ expected: 'HTML And CSS API'
+ },
+ {
+ description: 'slug with all supported acronyms',
+ slug: 'ai-cli-mcp-uxd-ui-faq-faqs-aria-rtl',
+ section: undefined,
+ expected: 'AI CLI MCP UXD UI FAQ FAQS ARIA RTL'
+ },
+ {
+ description: 'compound slug with underscore separator',
+ slug: 'ai-assisted-development_ai-assisted-code-migration',
+ section: undefined,
+ expected: 'AI Assisted Development: AI Assisted Code Migration'
+ },
+ {
+ description: 'compound slug with multiple underscore sections',
+ slug: 'section-one_section-two_section-three',
+ section: undefined,
+ expected: 'Section One: Section Two: Section Three'
+ },
+ {
+ description: 'overview slug with section',
+ slug: 'overview',
+ section: 'components',
+ expected: 'Components Overview'
+ },
+ {
+ description: 'overview slug with section containing acronym',
+ slug: 'overview',
+ section: 'ai-assist',
+ expected: 'AI Assist Overview'
+ },
+ {
+ description: 'overview slug with multi-word section',
+ slug: 'overview',
+ section: 'user-interface-patterns',
+ expected: 'User Interface Patterns Overview'
+ },
+ {
+ description: 'overview slug without section',
+ slug: 'overview',
+ section: undefined,
+ expected: 'Overview'
+ }
+ ])('should format slug to title, $description', ({ slug, section, expected }) => {
+ expect(formatSlugToTitle(slug, section)).toBe(expected);
+ });
+});
+
+describe('extractApiDisplayName', () => {
+ it.each([
+ {
+ description: 'props category with valid JSON containing name property',
+ content: JSON.stringify({ name: 'ButtonProps', props: {} }),
+ context: { category: 'props', slug: 'button-props' },
+ expected: 'ButtonProps'
+ },
+ {
+ description: 'props category with valid JSON without name property falls back to slug',
+ content: JSON.stringify({ props: {} }),
+ context: { category: 'props', slug: 'button-props' },
+ expected: 'Button Props'
+ },
+ {
+ description: 'props category with invalid JSON falls back to slug',
+ content: '{ invalid json',
+ context: { category: 'props', slug: 'button-props' },
+ expected: 'Button Props'
+ },
+ {
+ description: 'props category with markdown H1 heading',
+ content: '# Custom Button Props\nSome description',
+ context: { category: 'props', slug: 'button-props' },
+ expected: 'Custom Button Props'
+ },
+ {
+ description: 'css category with slug not containing CSS appends CSS',
+ content: '[]',
+ context: { category: 'css', slug: 'button', section: 'components' },
+ expected: 'Button CSS'
+ },
+ {
+ description: 'css category with slug already containing CSS does not append CSS',
+ content: '[]',
+ context: { category: 'css', slug: 'button-CSS', section: 'components' },
+ expected: 'Button CSS'
+ },
+ {
+ description: 'markdown content with H1 heading',
+ content: '# Card Component\nDescription text',
+ context: { slug: 'card' },
+ expected: 'Card Component'
+ },
+ {
+ description: 'markdown content with H1 overview and section',
+ content: '# Overview\nOverview body',
+ context: { slug: 'overview', section: 'components' },
+ expected: 'Components Overview'
+ },
+ {
+ description: 'markdown content with H1 Overview without section',
+ content: '# Overview\nOverview body',
+ context: { slug: 'overview' },
+ expected: 'Overview'
+ },
+ {
+ description: 'markdown content without H1 heading falls back to slug',
+ content: '## Subheading\nParagraph text without H1',
+ context: { slug: 'data-list', section: 'components' },
+ expected: 'Data List'
+ },
+ {
+ description: 'empty content falls back to slug and section',
+ content: '',
+ context: { slug: 'alert-group', section: 'components' },
+ expected: 'Alert Group'
+ },
+ {
+ description: 'undefined content and context return default',
+ content: undefined,
+ context: undefined,
+ expected: 'PatternFly API'
+ },
+ {
+ description: 'null context explicitly passed returns default',
+ content: undefined,
+ context: null as any,
+ expected: 'PatternFly API'
+ }
+ ])('should extract API display name, $description', ({ content, context, expected }: any) => {
+ expect(extractApiDisplayName(content, context)).toBe(expected);
+ });
+});
+
+describe('getApiFallbackDescription', () => {
+ it.each([
+ {
+ description: 'props category',
+ displayName: 'Button',
+ category: 'props',
+ expected: 'PatternFly React component props and TypeScript interfaces for Button.'
+ },
+ {
+ description: 'css category without css in displayName',
+ displayName: 'Button',
+ category: 'css',
+ expected: 'PatternFly CSS variables and tokens for Button.'
+ },
+ {
+ description: 'css category with uppercase CSS in displayName',
+ displayName: 'Button CSS',
+ category: 'css',
+ expected: 'PatternFly variables and tokens for Button CSS.'
+ },
+ {
+ description: 'css category with lowercase css in displayName',
+ displayName: 'button css tokens',
+ category: 'css',
+ expected: 'PatternFly variables and tokens for button css tokens.'
+ },
+ {
+ description: 'html category',
+ displayName: 'Button',
+ category: 'html',
+ expected: 'PatternFly HTML examples and markup structure for Button.'
+ },
+ {
+ description: 'html-demos category',
+ displayName: 'Card',
+ category: 'html-demos',
+ expected: 'PatternFly HTML examples and markup structure for Card.'
+ },
+ {
+ description: 'react category',
+ displayName: 'Button',
+ category: 'react',
+ expected: 'PatternFly React component examples and demos for Button.'
+ },
+ {
+ description: 'react-demos category',
+ displayName: 'Modal',
+ category: 'react-demos',
+ expected: 'PatternFly React component examples and demos for Modal.'
+ },
+ {
+ description: 'examples category',
+ displayName: 'Button',
+ category: 'examples',
+ expected: 'PatternFly Button examples and demos.'
+ },
+ {
+ description: 'doc category',
+ displayName: 'Button',
+ category: 'doc',
+ expected: 'PatternFly documentation and guidelines for Button.'
+ },
+ {
+ description: 'unrecognized category defaults to doc format',
+ displayName: 'Button',
+ category: 'custom',
+ expected: 'PatternFly documentation and guidelines for Button.'
+ },
+ {
+ description: 'default arguments without parameters',
+ displayName: undefined,
+ category: undefined,
+ expected: 'PatternFly documentation and guidelines for .'
+ }
+ ])('should provide fallback description, $description', ({ displayName, category, expected }: any) => {
+ expect(getApiFallbackDescription(displayName, category)).toBe(expected);
+ });
+});
+
+describe('extractApiDescription', () => {
+ it.each([
+ {
+ description: 'props category returns fallback',
+ content: '# Title\nValid paragraph line exceeding twenty characters in length.',
+ context: { displayName: 'Button', category: 'props' },
+ expected: 'PatternFly React component props and TypeScript interfaces for Button.'
+ },
+ {
+ description: 'css category returns fallback',
+ content: '# Title\nValid paragraph line exceeding twenty characters in length.',
+ context: { displayName: 'Button', category: 'css' },
+ expected: 'PatternFly CSS variables and tokens for Button.'
+ },
+ {
+ description: 'detailType equals examples returns fallback',
+ content: '# Title\nValid paragraph line exceeding twenty characters in length.',
+ context: { displayName: 'Button', detailType: 'examples' },
+ expected: 'PatternFly Button examples and demos.'
+ },
+ {
+ description: 'markdown content uses first valid paragraph',
+ content: '# Button\n\nA button is a clickable interactive element that triggers an action.',
+ context: { displayName: 'Button', category: 'doc' },
+ expected: 'A button is a clickable interactive element that triggers an action.'
+ },
+ {
+ description: 'markdown formatting like bold, italics, inline code, and links are stripped',
+ content: '# Title\n\nA **button** communicates an [action](https://patternfly.org) to be *performed* with `onClick`.',
+ context: { displayName: 'Button', category: 'doc' },
+ expected: 'A button communicates an action to be performed with onClick.'
+ },
+ {
+ description: 'HTML links and tags are stripped or converted to inner text',
+ content: '# Title\n\nA button with badge indicator.',
+ context: { displayName: 'Button', category: 'doc' },
+ expected: 'A button with badge indicator.'
+ },
+ {
+ description: 'trailing colon in paragraph is replaced with period',
+ content: '# Title\n\nHere is a list of components and features available for use:',
+ context: { displayName: 'Button', category: 'doc' },
+ expected: 'Here is a list of components and features available for use.'
+ },
+ {
+ description: 'paragraph exceeding 200 characters is truncated',
+ content: `# Title\n\n${'This is a long description sentence describing the component. '.repeat(5)}`,
+ context: { displayName: 'Button', category: 'doc' },
+ expected: `${'This is a long description sentence describing the component. '.repeat(5).trim().slice(0, 197)}...`
+ },
+ {
+ description: 'content with imports and code blocks filtered out before extracting prose',
+ content: "import React from 'react';\n```tsx\n\n```\n# Heading\n| table |\n\nA valid paragraph with more than twenty characters for description.",
+ context: { displayName: 'Button', category: 'doc' },
+ expected: 'A valid paragraph with more than twenty characters for description.'
+ },
+ {
+ description: 'content without valid prose paragraphs uses fallback',
+ content: "import React from 'react';\n# Heading\nShort line",
+ context: { displayName: 'Card', category: 'doc' },
+ expected: 'PatternFly documentation and guidelines for Card.'
+ },
+ {
+ description: 'undefined content uses fallback',
+ content: undefined,
+ context: { displayName: 'Card', category: 'doc' },
+ expected: 'PatternFly documentation and guidelines for Card.'
+ },
+ {
+ description: 'undefined content and undefined context returns fallback',
+ content: undefined,
+ context: undefined,
+ expected: 'PatternFly documentation and guidelines for .'
+ },
+ {
+ description: 'null context explicitly passed returns fallback',
+ content: undefined,
+ context: null as any,
+ expected: 'PatternFly documentation and guidelines for .'
+ }
+ ])('should extract API description, $description', ({ content, context, expected }: any) => {
+ expect(extractApiDescription(content, context)).toBe(expected);
+ });
+});
+
+describe('extractApiName', () => {
+ it.each([
+ {
+ description: 'components section returns normalized item name',
+ item: 'Button',
+ section: 'components',
+ expected: 'button'
+ },
+ {
+ description: 'components section with uppercase and whitespace',
+ item: ' Card ',
+ section: ' Components ',
+ expected: 'card'
+ },
+ {
+ description: 'overview item with custom section adds suffix',
+ item: 'overview',
+ section: 'utilities',
+ expected: 'utilities-overview'
+ },
+ {
+ description: 'item already prefixed with section avoids double prefix',
+ item: 'charts-pie',
+ section: 'charts',
+ expected: 'charts-pie'
+ },
+ {
+ description: 'item already prefixed with uppercase section name',
+ item: 'Patterns-Gallery',
+ section: 'patterns',
+ expected: 'patterns-gallery'
+ },
+ {
+ description: 'non-prefixed item in custom section prefixes section',
+ item: 'pie',
+ section: 'charts',
+ expected: 'charts-pie'
+ },
+ {
+ description: 'non-prefixed item in patterns section prefixes section',
+ item: 'gallery',
+ section: 'patterns',
+ expected: 'patterns-gallery'
+ }
+ ])('should extract API name, $description', ({ item, section, expected }) => {
+ expect(extractApiName(item, section)).toBe(expected);
+ });
+});
diff --git a/src/collection.patternFlyApi.ts b/src/collection.patternFlyApi.ts
index 044ab9d0..fb194209 100644
--- a/src/collection.patternFlyApi.ts
+++ b/src/collection.patternFlyApi.ts
@@ -6,7 +6,7 @@ import {
import { log } from './logger';
import { processDocsFunction } from './server.getResources';
import { memo } from './server.caching';
-import { isPlainObject, joinUrl } from './server.helpers';
+import { isPlainObject, joinUrl, timeoutFunction } from './server.helpers';
import {
getOptions,
getSessionOptions,
@@ -14,31 +14,48 @@ import {
runWithSession
} from './options.context';
import { DEFAULT_OPTIONS } from './options.defaults';
+import {
+ calculateContentQualityScore,
+ extractApiDescription,
+ extractApiDisplayName,
+ extractApiName,
+ normalizeSlug
+} from './collection.patternFlyApiHelpers';
+import { contentType } from './resource.helpers';
/**
* Processed content for API responses.
*
- * @property url - The URL of the content.
- * @property content - The content itself.
- * @property semanticContext - Semantic context of the content.
- * @property semanticContext.version - PatternFly version of the content.
- * @property semanticContext.section - Section of the content.
- * @property semanticContext.item - Item of the content.
- * @property semanticContext.facet - Facet of the content.
- * @property semanticContext.kind - Kind of the content.
- * @property semanticContext.metadata - Remaining metadata, if any, of the content.
+ * @interface ApiContent
+ *
+ * @property description - Description of the content.
+ * @property displayName - Display name of the content.
+ * @property category - Category of the content.
+ * @property isLowQuality - Whether the content is low quality.
+ * @property id - ID of the content.
+ * @property isDeferred - Whether the content is deferred.
+ * @property name - Name of the content.
+ * @property path - Path of the content.
+ * @property pathSlug - Slug path of the content.
+ * @property section - Section of the content.
+ * @property source - Source of the content.
+ * @property version - Version of the content.
*/
interface ApiContent {
- url: string;
+ description: string;
+ displayName: string;
+ category: string;
content: string;
- semanticContext: {
- version?: string | undefined;
- section?: string | undefined;
- item?: string | undefined;
- facet?: string | undefined;
- kind?: string | undefined;
- metadata?: string[] | undefined;
- }
+ contentType: string;
+ isLowQuality: boolean;
+ id: string;
+ isDeferred: boolean;
+ name: string;
+ path: string;
+ pathSlug: string;
+ section: string;
+ source: string;
+ version: string;
}
/**
@@ -74,6 +91,36 @@ interface ParsePayload {
payload: ParsePayloadApi;
}
+/**
+ * Deferred API categories.
+ *
+ * @note Minimal PatternFly API data quality threshold
+ * - Last resort for content that requires additional parsing or should be ignored.
+ * - A quality threshold still has to be met even if these items are removed
+ * - Quality metrics need to be updated periodically as API content is added.
+ *
+ * - `props`: Deferred in favor of using @patternfly/patternfly-component-schemas.
+ * - `react`: Quality threshold applied. Some examples still contain low-quality data.
+ * - `react-demos`: Deferred React demonstration components.
+ * - `html`: Quality threshold applied. Some examples still contain low-quality data.
+ * - `html-demos`: Deferred HTML demonstration examples.
+ * - `text`: Quality threshold applied. Some examples still contain low-quality data.
+ */
+const DEFERRED_API_CATEGORIES = new Set([
+ 'props',
+ // 'react',
+ 'react-demos',
+ // 'html',
+ 'html-demos'
+ // 'text',
+ // 'examples'
+]);
+
+/**
+ * Min content quality threshold. See {@link calculateContentQualityScore}
+ */
+const MIN_API_QUALITY_THRESHOLD = 0.95;
+
/**
* Parses the given payload and determines its state and structure.
*
@@ -85,12 +132,19 @@ interface ParsePayload {
* Otherwise, the trimmed string or original value is provided.
*/
const parsePayload = (payload: unknown): ParsePayload => {
- const updatedPayload = typeof payload === 'string' ? payload.trim() : '';
+ let updatedPayload: string | Record = '';
+
+ if (typeof payload === 'string') {
+ updatedPayload = payload.trim();
+ } else if (isPlainObject(payload)) {
+ updatedPayload = payload;
+ }
+
let isEmpty: boolean;
let parsedPayload: ParsePayloadApi;
try {
- parsedPayload = JSON.parse(updatedPayload);
+ parsedPayload = typeof updatedPayload === 'string' ? JSON.parse(updatedPayload) : updatedPayload;
if (typeof parsedPayload === 'number') {
isEmpty = false;
@@ -133,43 +187,113 @@ const isEmptyPayload = (payload: unknown) => {
*/
isEmptyPayload.memo = memo(isEmptyPayload, DEFAULT_OPTIONS.resourceMemoOptions.default);
+/**
+ * Filters and returns a list of unique URLs from the input array, ensuring no duplicates.
+ *
+ * @param urls - Array of URLs to be filtered for uniqueness.
+ * @param [visited] - Set object for visited URLs. Defaults to an empty Set.
+ * @returns An array containing only unique URLs from the input array.
+ */
+const getUniqueUrls = (urls: string[], visited = new Set()) => urls.filter(url => {
+ if (visited.has(url)) {
+ return false;
+ }
+
+ visited.add(url);
+
+ return true;
+});
+
/**
* Recursively crawls a list of URLs.
*
* Resolves paths and fetches content; built specifically around the PatternFly API response structure.
*
* @param urls - The list of URLs to crawl.
+ * @param [settings] - An optional configuration object.
+ * @param [settings.visited] - Used to track visited paths.
+ * @param [settings.signal] - AbortSignal for the crawling operation.
* @param [options] - An optional configuration object.
- * @returns {Promise} A promise that resolves to an array of processed documents,
+ * @returns {Promise} A promise that resolves to an array of processed documents,
* each containing information about the crawling result, status, and content.
*/
-const crawler = async (urls: string[], options = getOptions()): Promise => {
- const componentPaths = options.patternflyOptions.api.componentPaths;
- const settled = await processDocsFunction(urls);
+const crawler = async (
+ urls: string[],
+ { visited = new Set(), signal }: { visited?: Set; signal?: AbortSignal | undefined } = {},
+ options = getOptions()
+): Promise => {
+ if (signal?.aborted) {
+ log.debug('Aborted PatternFly API collection crawl.');
+
+ return [];
+ }
+
+ const { componentPaths, traversalPaths } = options.patternflyOptions.api;
+ const uniqueUrls = getUniqueUrls(urls, visited);
+
+ if (uniqueUrls.length === 0) {
+ return [];
+ }
+
+ const settled = await processDocsFunction(uniqueUrls) || [];
const content: ApiCrawler[] = [];
for (const res of settled) {
+ if (!res.isSuccess) {
+ continue;
+ }
+
const { isEmpty, payload } = parsePayload.memo(res.content);
- if (res.isSuccess) {
- if (Array.isArray(payload)) {
- if (componentPaths.some(componentPath => res?.path?.includes(componentPath))) {
- if (!isEmpty) {
- content.push({ ...res });
- }
- continue;
+ if (Array.isArray(payload)) {
+ // Terminal Data Arrays (props, css, etc)
+ if (componentPaths.some(componentPath => res?.path?.endsWith(`/${componentPath}`))) {
+ if (!isEmpty) {
+ content.push({ ...res });
}
-
- const updatedPayload = [...payload, ...componentPaths].map(path => joinUrl(res.path, path));
- const crawledContent = await crawler(updatedPayload);
-
- content.push(...crawledContent);
continue;
}
- if (!isEmpty) {
- content.push({ ...res });
- }
+ // Traversal & Directory Array Processing
+ const flattenedPayload: string[] = [];
+
+ payload.forEach(value => {
+ if (typeof value === 'string') {
+ flattenedPayload.push(value);
+
+ log.debug(`Collection PatternFly API adding path`, value);
+ } else if (isPlainObject(value)) {
+ Object.values(value).forEach(value => {
+ if (typeof value === 'string') {
+ flattenedPayload.push(value);
+
+ log.debug(`Collection PatternFly API adding path`, value);
+ }
+ });
+ }
+ });
+
+ const updatedPayload = [...flattenedPayload, ...traversalPaths, ...componentPaths].map(path => joinUrl(res.path, path));
+
+ log.debug(`Collection PatternFly API Crawling ${updatedPayload.length} path(s)`);
+
+ const crawledContent = await crawler(updatedPayload, { visited, signal });
+
+ content.push(...crawledContent);
+ continue;
+ }
+
+ // String Payloads (Markdown, HTML, .tsx source code)
+ if (!isEmpty) {
+ content.push({ ...res });
+ }
+
+ // Probe Traversal Paths on Facet Endpoints (e.g. /react -> /react/examples)
+ if (!traversalPaths.some(traversalPath => res?.path?.endsWith(`/${traversalPath}`))) {
+ const traversalUrls = traversalPaths.map(traversalPath => joinUrl(res.path, traversalPath));
+ const traversalCrawledContent = await crawler(traversalUrls, { visited, signal });
+
+ content.push(...traversalCrawledContent);
}
}
@@ -208,48 +332,16 @@ const getVersions = async (options = getOptions()) => {
return versions;
};
-/**
- * Process content metadata from response paths.
- *
- * @param apiResponses - The list of pre-metadata content.
- * @param [options=getOptions()] - Configuration options.
- * @returns The list of processed API content with metadata.
- */
-const contentMetadata = (apiResponses: ApiCrawler[], options = getOptions()): ApiContent[] => {
- const base = options.patternflyOptions.api.base;
- const componentPaths = options.patternflyOptions.api.componentPaths;
-
- return apiResponses.map(({ content, resolvedPath }) => {
- const [version, section, item, facet, ...remaining] = resolvedPath.replace(base, '').split('/').filter(Boolean) || [];
- const kind = facet && (componentPaths.includes(facet) || remaining.includes(facet)) ? facet : 'doc';
-
- return {
- url: resolvedPath,
- content,
- semanticContext: {
- version,
- section,
- item,
- facet,
- kind,
- metadata: (remaining.length && remaining) || undefined
- }
- };
- });
-};
-
-/**
- * Memoized version of contentMetadata.
- */
-contentMetadata.memo = memo(contentMetadata);
-
/**
* Initiate API crawl.
*
+ * @param options - Options for the API spider.
* @returns A promise resolving to an array of processed API content entries.
*/
-const apiSpider = async (): Promise => {
- log.info(`API spider crawl started`);
+const apiSpider = async (options = getOptions()): Promise => {
+ log.info(`Collection PatternFly API spider crawl started`);
+
+ const { timeoutMs } = options.patternflyOptions.api;
let seedVersions: string[] = [];
let content: ApiCrawler[] = [];
@@ -262,30 +354,99 @@ const apiSpider = async (): Promise => {
}
if (seedVersions.length) {
+ const controller = new AbortController();
+
try {
- content = await crawler(seedVersions);
+ content = await timeoutFunction(
+ () => crawler(seedVersions, { visited: new Set(), signal: controller.signal }),
+ {
+ timeout: timeoutMs,
+ errorMessage: `Crawl timed out after ${timeoutMs}ms`
+ }
+ );
} catch (err) {
- log.warn(`API spider: crawler failed`, err);
+ controller.abort();
+ log.warn(`Collection PatternFly API spider: crawler failed`, err);
return [];
}
}
- // Review the memo here. It may be better served to tie into crawler,
- // like `crawler.memo` as part of the countdown to refresh
- const updatedContent = contentMetadata.memo(content);
-
log.info(
- `API spider crawl completed. ${updatedContent.length} content ${
- (updatedContent.length === 1 && 'entry') || 'entries'
+ `Collection PatternFly API spider crawl completed. ${content.length} content ${
+ (content.length === 1 && 'entry') || 'entries'
} retrieved.`
);
- return updatedContent;
+ return content;
+};
+
+/**
+ * Light/Immediate process for content metadata from response paths.
+ *
+ * @param crawlerResponse - An entry with pre-metadata content.
+ * @param [options] - Configuration options.
+ * @returns The process metadata entry.
+ */
+const contentMetadata = (crawlerResponse: ApiCrawler, options = getOptions()): ApiContent => {
+ const { content, resolvedPath } = crawlerResponse;
+ const { base } = options.patternflyOptions.api;
+
+ // Relative path after '/api/'
+ const segments = resolvedPath.replace(base, '').split('/').filter(Boolean);
+ const [version = 'unknown', section = 'components', rawItem = 'api-entry', rawFacet = 'doc', rawDetailType = '', rawDetail = '', ...remaining] = segments;
+
+ const normalizedVersion = version.toLowerCase();
+ const normalizedSection = normalizeSlug(section);
+ const normalizedItem = normalizeSlug(rawItem);
+ const normalizedFacet = normalizeSlug(rawFacet);
+ const normalizedDetailType = normalizeSlug(rawDetailType);
+ const normalizedDetail = normalizeSlug(rawDetail);
+
+ // Make a category from the normalized facet
+ const normalizedCategory = normalizedFacet;
+
+ // Build hierarchical normalized path slug: e.g. "AI/overview/text" or "components/button/props"
+ const isDetailSameName = normalizedDetail && normalizedDetail.includes(normalizedItem);
+ const pathSlug = [
+ normalizedSection,
+ isDetailSameName ? undefined : normalizedItem,
+ normalizedFacet,
+ normalizedDetailType,
+ normalizedDetail,
+ ...remaining.map(normalizeSlug)
+ ].filter(Boolean).join('-');
+
+ const name = extractApiName(normalizedItem, normalizedSection);
+
+ const id = `api::${normalizedVersion}::${normalizedSection}::${normalizedItem}::${normalizedCategory}${normalizedDetailType ? `::${normalizedDetailType}::${normalizedDetail}` : ''}`;
+
+ const displayName = extractApiDisplayName(content, { slug: normalizedItem, category: normalizedCategory, section: normalizedSection });
+ const description = extractApiDescription(content, { displayName, category: normalizedCategory, detailType: normalizedDetailType });
+
+ const isLowQuality = calculateContentQualityScore(content, { category: normalizedCategory }) < MIN_API_QUALITY_THRESHOLD;
+ const isDeferred = DEFERRED_API_CATEGORIES.has(normalizedCategory);
+
+ return {
+ description,
+ displayName,
+ category: normalizedCategory,
+ content,
+ contentType: contentType(content),
+ isLowQuality,
+ id,
+ isDeferred,
+ name,
+ path: resolvedPath,
+ pathSlug,
+ section: normalizedSection,
+ source: 'api' as const,
+ version: normalizedVersion
+ };
};
/**
- * Async collect and process entries for a collection.
+ * Async collect and process entries for a collection. Add "conditional" metadata.
*
* @returns {Promise} Object containing a list of processed records.
*/
@@ -293,41 +454,30 @@ const collectionCallback = async (): Promise => {
const entries = await apiSpider();
const recordsMap: Map = new Map();
- entries?.forEach((entry, index) => {
- const semanticContext = entry.semanticContext || {};
- const name = (semanticContext.item || 'api-entry').toLowerCase();
- const version = (semanticContext.version || 'unknown').toLowerCase();
- const displayName = semanticContext.item || name;
+ for (const entry of entries) {
+ const { name, isDeferred, isLowQuality, ...metadata } = contentMetadata(entry);
- const id = `api::${version}::${semanticContext.section || ''}::${name}::${semanticContext.kind || ''}::${index}`;
-
- if (recordsMap.has(id)) {
- return;
+ if (isDeferred || isLowQuality) {
+ continue;
}
- const adaptedEntry = {
- displayName,
- description: entry.content || `PatternFly API documentation for ${displayName}`,
- pathSlug: name,
- category: semanticContext.kind,
- section: semanticContext.section || 'components',
- source: 'api' as const,
- version,
- id,
- path: entry.url
- };
+ if (recordsMap.has(metadata.id)) {
+ continue;
+ }
const record = {
- id,
- sourceId: entry.url,
+ id: metadata.id,
+ sourceId: metadata.path,
sourceType: 'api' as const,
data: {
- [name]: adaptedEntry
+ [name]: [{
+ ...metadata
+ }]
}
};
recordsMap.set(record.id, record);
- });
+ }
return { records: [...recordsMap.values()] };
};
@@ -350,8 +500,7 @@ const patternFlyApiCollection = (options = getOptions(), session = getSessionOpt
{
runParallel: '#collectionPatternFlyApi',
runSchedule: {
- cancelMs: options.patternflyOptions.api.crawlCancelMs,
- intervalMs: options.patternflyOptions.api.crawlIntervalMs
+ ...options.patternflyOptions.api.schedule
}
}
];
@@ -362,6 +511,7 @@ export {
collectionCallback,
apiSpider,
crawler,
+ getUniqueUrls,
isEmptyPayload,
parsePayload,
type ApiContent,
diff --git a/src/collection.patternFlyApiHelpers.ts b/src/collection.patternFlyApiHelpers.ts
new file mode 100644
index 00000000..53052cb3
--- /dev/null
+++ b/src/collection.patternFlyApiHelpers.ts
@@ -0,0 +1,389 @@
+import { isJson, isJsonLike } from './resource.helpers';
+
+/**
+ * Detect imports that use the `?raw` query param.
+ *
+ * @param str
+ */
+const isRawImport = (str: string) =>
+ /import\s+[\w*\s{},]+\s+from\s+['"][^'"]+\?raw['"]/i.test(str);
+
+/**
+ * Detect a `` tag.
+ *
+ * @param str
+ */
+const hasLiveExample = (str: string) => /]*\/?>/i.test(str);
+
+/**
+ * Count the number of `` tags in a given string.
+ *
+ * @param str - Input string to search for `` tags.
+ * @returns `` count found in the input string.
+ */
+const getLiveExampleCount = (str: string) =>
+ (str.match(/]*\/?>/gi) || []).length;
+
+/**
+ * Detect empty code fences with external file references that weren't
+ * inlined. (e.g., ```ts file = "./ButtonBasic.tsx" \n```)
+ *
+ * Considered empty if:
+ * - A fenced code block with a `file` attribute is specified but no content.
+ * - A fenced code block with no content inside the block, regardless of attributes or language.
+ *
+ * @param str - Input string.
+ * @returns Returns `true` if the input string contains an empty code fence.
+ */
+const hasEmptyFileCodeFence = (str: string) =>
+ /```[\w-]*\s+file="[^"]+"\s*\n\s*```/i.test(str) ||
+ /```[\w-]*\s*\n\s*```/.test(str);
+
+/**
+ * Calculate a quality score for a PatternFly API response.
+ *
+ * @param content - Content to score.
+ * @param [options] - Function options
+ * @param [options.baseScore] - Base starting score.
+ * @param [options.category] - Used to determine which quality metrics are applied.
+ * @param [options.qualityReduction] - Amount to reduce the base score for each quality metric.
+ * @param [options.minCharacters] - Minimum number of characters required to avoid quality reduction.
+ * @returns The calculated quality score.
+ */
+const calculateContentQualityScore = (
+ content: unknown,
+ {
+ baseScore = 1, category, qualityReduction = 0.03, minCharacters = 150
+ }: { baseScore?: number; category?: undefined | string; qualityReduction?: number; minCharacters?: number } = {}
+): number => {
+ if (content === undefined || content === null) {
+ return baseScore;
+ }
+
+ const raw = typeof content === 'number' ? String(content) : content;
+
+ if (typeof raw !== 'string') {
+ return baseScore;
+ }
+
+ const trimmed = raw.trim();
+
+ if (trimmed.length === 0) {
+ return baseScore;
+ }
+
+ if (category === 'examples') {
+ return baseScore;
+ }
+
+ let score = baseScore;
+
+ if (isJsonLike(trimmed)) {
+ const jsonValid = isJson(trimmed);
+
+ if (!jsonValid) {
+ score -= qualityReduction;
+ }
+ }
+
+ if (isRawImport(trimmed)) {
+ score -= qualityReduction;
+ }
+
+ if (hasLiveExample(trimmed)) {
+ score -= qualityReduction * getLiveExampleCount(trimmed);
+ }
+
+ if (trimmed.length < minCharacters && !trimmed.includes('```') && !hasEmptyFileCodeFence(trimmed)) {
+ score -= qualityReduction;
+ }
+
+ if (hasEmptyFileCodeFence(trimmed)) {
+ score -= qualityReduction;
+
+ if (trimmed.length < minCharacters) {
+ score -= qualityReduction;
+ }
+ }
+
+ return Number(Math.min(1, Math.max(0, score)).toFixed(3));
+};
+
+/**
+ * Transform a string.
+ *
+ * @param segment - Input string to normalize.
+ * @returns Normalized slug.
+ */
+const normalizeSlug = (segment: string): string => {
+ let updatedSegment = segment;
+
+ if (/[A-Z]/.test(updatedSegment) && !/^(ai|css|html|mcp|cli|uxd|ui|api|faq|faqs|aria|rtl)$/i.test(updatedSegment)) {
+ const split = updatedSegment.split(/(?=[A-Z])/);
+
+ if (split.every(val => /^[A-Z]/.test(val))) {
+ updatedSegment = split.join('-');
+ }
+ }
+
+ return updatedSegment
+ .trim()
+ .toLowerCase()
+ .replace(/_/g, '-')
+ .replace(/-+/g, '-');
+};
+
+/**
+ * Format a compound slug into a clean title.
+ * E.g., 'ai-assisted-development_ai-assisted-code-migration' -> 'AI Assisted Development: AI Assisted Code Migration'
+ *
+ * @param slug
+ * @param section
+ */
+const formatSlugToTitle = (slug: string, section?: string): string => {
+ if (!slug) {
+ return 'PatternFly API';
+ }
+
+ const acronyms = ['ai', 'css', 'html', 'mcp', 'cli', 'uxd', 'ui', 'api', 'faq', 'faqs', 'aria', 'rtl'];
+ const acronymRegex = new RegExp(`^(${acronyms.join('|')})$`, 'i');
+
+ const cleanSection = section
+ ? section
+ .split('-')
+ .map(wordPhrase =>
+ (acronymRegex.test(wordPhrase)
+ ? wordPhrase.toUpperCase()
+ : wordPhrase.charAt(0).toUpperCase() + wordPhrase.slice(1))).join(' ')
+ : '';
+
+ // Handle bare generic names like 'overview'
+ if (slug.toLowerCase() === 'overview' && cleanSection) {
+ return `${cleanSection} Overview`;
+ }
+
+ return slug
+ .split('_')
+ .map(segment =>
+ segment
+ .split('-')
+ .map(word => {
+ if (acronymRegex.test(word)) {
+ return word.toUpperCase();
+ }
+
+ return word.charAt(0).toUpperCase() + word.slice(1);
+ })
+ .join(' '))
+ .join(': ');
+};
+
+/**
+ * Generate a display name from metadata.
+ *
+ * @param [content] - Optional content string.
+ * @param [context] - Optional context object for generating a unique name.
+ * @param [context.slug] - Optional slug used for fallback or secondary formatting of the display name.
+ * @param [context.category] - Optional category of content being processed (e.g., 'props', 'css', or 'doc').
+ * @param [context.section] - Optional section name used for refining the display name.
+ * @returns Extracted or formatted display name for the API item.
+ */
+const extractApiDisplayName = (content?: string, context: { slug?: string; category?: string; section?: string; } = {}): string => {
+ const { slug = '', category = 'doc', section } = context || {};
+
+ const trimmed = content?.trim() || '';
+
+ // Props JSON signature
+ if (category === 'props' && trimmed.startsWith('{')) {
+ try {
+ const parsed = JSON.parse(trimmed);
+
+ if (parsed.name) {
+ return parsed.name;
+ }
+ } catch {}
+ }
+
+ // CSS JSON Array signature
+ if (category === 'css') {
+ return slug.toLowerCase().includes('css') ? formatSlugToTitle(slug, section) : `${formatSlugToTitle(slug, section)} CSS`;
+ }
+
+ // Markdown H1 signature (# Title)
+ const h1Match = trimmed.match(/^#\s+([^\r\n]+)/m);
+
+ if (h1Match?.[1]?.trim()) {
+ const title = h1Match[1].trim();
+
+ // If the H1 is just "Overview", qualify it with the section
+ if (title.toLowerCase() === 'overview' && section) {
+ return formatSlugToTitle('overview', section);
+ }
+
+ return title;
+ }
+
+ // Fallback to slug
+ return formatSlugToTitle(slug, section);
+};
+
+/**
+ * Provide a fallback description based on kind/category when no prose is available.
+ *
+ * @param displayName - Display name
+ * @param category - Category / facet kind
+ */
+const getApiFallbackDescription = (displayName = '', category = 'doc'): string => {
+ switch (category) {
+ case 'props':
+ return `PatternFly React component props and TypeScript interfaces for ${displayName}.`;
+ case 'css':
+ return `PatternFly ${
+ displayName.toLowerCase().includes('css') ? '' : 'CSS '}variables and tokens for ${displayName}.`;
+ case 'html':
+ case 'html-demos':
+ return `PatternFly HTML examples and markup structure for ${displayName}.`;
+ case 'react':
+ case 'react-demos':
+ return `PatternFly React component examples and demos for ${displayName}.`;
+ case 'examples':
+ return `PatternFly ${displayName} examples and demos.`;
+ default:
+ return `PatternFly documentation and guidelines for ${displayName}.`;
+ }
+};
+
+/**
+ * Generate a description from metadata.
+ *
+ * @param [content] - Optional content.
+ * @param [context] - Optional context for generating a unique description.
+ * @param [context.displayName] - Display name.
+ * @param [context.category] - Type of content.
+ * @param [context.detailType] - Alternate to `category, like "examples".
+ * @returns A generated description from metadata, or a fallback.
+ */
+const extractApiDescription = (
+ content?: string,
+ context: { displayName?: string; category?: string; detailType?: string | undefined } = {}
+): string => {
+ const { displayName = '', category = 'doc', detailType = '' } = context || {};
+
+ if (category === 'props' || category === 'css') {
+ return getApiFallbackDescription(displayName, category);
+ }
+
+ if (detailType === 'examples') {
+ return getApiFallbackDescription(displayName, detailType);
+ }
+
+ if (content) {
+ // Replace import statements, multiline code blocks
+ const cleanContent = content
+ .replace(/import\s+[\s\S]*?from\s+['"][^'"]+['"];?/gm, '')
+ .replace(/import\s+['"][^'"]+['"];?/gm, '')
+ .replace(/```[\s\S]*?```/gm, '');
+
+ // Filter headings, tags, and common HTML attributes
+ const lines = cleanContent
+ .split('\n')
+ .map(line => line.trim())
+ .filter(line =>
+ line &&
+ !line.startsWith('import ') &&
+ !line.startsWith('#') &&
+ !line.startsWith('---') &&
+ !line.startsWith('![') &&
+ !line.startsWith('<') &&
+ !line.startsWith('```') &&
+ !line.startsWith('export ') &&
+ !line.startsWith('|') &&
+ !line.startsWith('class=') &&
+ !line.startsWith('className=') &&
+ !line.startsWith('style=') &&
+ !line.startsWith('d="') &&
+ !line.startsWith('viewBox=') &&
+ !/^[A-Za-z]+="(.*)"/.test(line) &&
+ !/^(ts|tsx|js|jsx|html)\s+/i.test(line) &&
+ !line.includes('file="./') &&
+ !line.startsWith('["') &&
+ !line.endsWith(',') &&
+ !/^[A-Za-z0-9]+\./.test(line) &&
+ !/^[A-Z][A-Za-z0-9]+,$/.test(line) &&
+ line.length > 20);
+
+ // Finally, does the copy exist?
+ if (lines.length > 0 && lines[0]) {
+ let cleanPara = lines[0]
+ // Convert HTML links to their inner text
+ .replace(/]*>(.*?)<\/a>/gi, '$1')
+ // Remove closing HTML tags
+ .replace(/<\/[A-Za-z0-9_-]+>/g, '')
+ // Convert bare tags
+ .replace(/<([A-Za-z0-9_\s-]+)>/g, '$1')
+ // Remove remaining complex HTML tags with attributes
+ .replace(/<[A-Za-z0-9_-]+\b[^>]*\/?>/g, '')
+ // Replace Markdown inline images: ` -> alt`
+ .replace(/!\[([^\]]*)\]\([^)]+\)/g, '$1')
+ // Replace Markdown links: `[text](url) -> text`
+ .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+ // Replace Markdown reference links: `[text][ref] -> text`
+ .replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1')
+ // Remove Markdown formatting characters (bold, italics, inline code, strikethrough)
+ .replace(/[*_`~]/g, '')
+ // Normalize excess whitespace
+ .replace(/\s+/g, ' ')
+ .trim();
+
+ if (cleanPara.endsWith(':')) {
+ cleanPara = `${cleanPara.slice(0, -1)}.`;
+ }
+
+ return cleanPara.length > 200 ? `${cleanPara.slice(0, 197)}...` : cleanPara;
+ }
+ }
+
+ // Fallback
+ return getApiFallbackDescription(displayName, category);
+};
+
+/**
+ * Extracts and constructs an API entry name based on the provided item and section.
+ *
+ * @param item - Entry base name.
+ * @param section - Entry section.
+ * @returns Extracted entry name
+ */
+const extractApiName = (item: string, section: string): string => {
+ const normalizedItem = item.trim().toLowerCase();
+ const normalizedSection = section.trim().toLowerCase();
+
+ if (normalizedSection === 'components') {
+ return normalizedItem;
+ }
+
+ if (normalizedItem === 'overview') {
+ return `${normalizedSection}-overview`;
+ }
+
+ // Prevent double-prefix
+ if (normalizedItem.startsWith(`${normalizedSection}-`)) {
+ return normalizedItem;
+ }
+
+ return `${normalizedSection}-${normalizedItem}`;
+};
+
+export {
+ calculateContentQualityScore,
+ extractApiDescription,
+ extractApiDisplayName,
+ extractApiName,
+ formatSlugToTitle,
+ getApiFallbackDescription,
+ getLiveExampleCount,
+ hasEmptyFileCodeFence,
+ hasLiveExample,
+ isRawImport,
+ normalizeSlug
+};
diff --git a/src/collections.ts b/src/collections.ts
index 78d3132e..f22e5856 100644
--- a/src/collections.ts
+++ b/src/collections.ts
@@ -64,7 +64,12 @@ type McpCollection = [
handler: (arg?: unknown) => McpCollectionResult | Promise,
_config?: {
runParallel?: `#${string}`;
- runSchedule?: { cancelMs?: number, intervalMs?: number };
+ runSchedule?: {
+ continueOnError?: boolean;
+ cancelMs?: number;
+ intervalMs?: number;
+ repeat?: number
+ };
// priority?: number;
isRequired?: boolean;
// group?: string;
diff --git a/src/options.defaults.ts b/src/options.defaults.ts
index d13f5e21..c20f2f91 100644
--- a/src/options.defaults.ts
+++ b/src/options.defaults.ts
@@ -181,9 +181,13 @@ interface ModeOptions {
* @property api PatternFly API.
* @property api.base URL starting base for crawling the PatternFly API.
* @property api.versions URL Get the available PatternFly API versions. Versions are required to crawl.
- * @property api.componentPaths List of additional PatternFly API component paths to try.
- * @property api.crawlCancelMs Timeout in milliseconds for cancelling the PatternFly API crawl.
- * @property api.crawlIntervalMs Interval in milliseconds, during server run, for crawling the PatternFly API.
+ * @property api.componentPaths List of additional PatternFly API component paths to try and terminate with expected content.
+ * @property api.traversalPaths List of additional PatternFly API traversal paths to iteratively try.
+ * @property api.timeoutMs Timeout in milliseconds, during server run, for crawling the PatternFly API.
+ * @property api.schedule Schedule for crawling the PatternFly API. See {@link McpCollection} config for details.
+ * @property api.schedule.continueOnError Continue crawling the PatternFly API on error.
+ * @property api.schedule.intervalMs Interval in milliseconds, during server run, for crawling the PatternFly API.
+ * @property api.schedule.repeat Number of times to repeat crawling the PatternFly API.
* @property availableResourceVersions List of available PatternFly resource versions to the MCP server.
* @property availableSearchVersions List of available PatternFly search versions to the MCP server.
* @property availableSchemasVersions List of available PatternFly schema versions to the MCP server.
@@ -201,9 +205,14 @@ interface PatternFlyOptions {
base: string;
versions: string;
componentPaths: string[];
- crawlCancelMs: number;
- crawlIntervalMs: number;
+ traversalPaths: string[];
+ timeoutMs: number;
enabled: boolean;
+ schedule: {
+ continueOnError: boolean;
+ intervalMs: number;
+ repeat: number;
+ }
},
availableResourceVersions: ('6.0.0')[];
availableSearchVersions: ('current' | 'latest' | 'v6')[];
@@ -517,10 +526,16 @@ const PATTERNFLY_OPTIONS: PatternFlyOptions = {
'props',
'css'
],
- crawlCancelMs: 180_000, // 3 minutes
- crawlIntervalMs: 43_200_000, // 12 hours
+ traversalPaths: [
+ 'examples'
+ ],
+ timeoutMs: 120_000,
+ schedule: {
+ continueOnError: true,
+ intervalMs: 86_400_000 * 7, // 7 days
+ repeat: Infinity
+ },
enabled: false
- // concurrency: 4
},
availableResourceVersions: ['6.0.0'],
availableSearchVersions: ['current', 'latest', 'v6'],
diff --git a/src/patternFly.getResources.ts b/src/patternFly.getResources.ts
index 0444ad78..633ad7bc 100644
--- a/src/patternFly.getResources.ts
+++ b/src/patternFly.getResources.ts
@@ -462,6 +462,20 @@ const mutateKeyWordsMap = (
mutateMap(normalizedKeyword);
};
+/**
+ * Normalizes collection resource names into a uniform slug.
+ *
+ * @param key - Raw catalog or collection identifier
+ * @returns Normalized slug for current resource grouping strategy.
+ */
+const normalizeKey = (key: string): string => {
+ if (!key) {
+ return '__unknown__';
+ }
+
+ return key.replace(/[^a-z0-9]/gi, '').toLowerCase();
+};
+
/**
* Get a multifaceted resources breakdown from PatternFly.
*
@@ -494,7 +508,7 @@ const getPatternFlyMcpResources = async (contextPathOverride?: string): Promise<
const rawKeywordsMap: PatternFlyMcpKeywordsMap = new Map();
catalog.forEach(([unifiedName, entries]) => {
- const name = unifiedName.toLowerCase();
+ const name = normalizeKey(unifiedName);
const groupId = generateHash(name);
hashIndexMap.set(groupId.toLowerCase(), name);
diff --git a/src/server.collections.ts b/src/server.collections.ts
index 31a5015f..cddf0370 100644
--- a/src/server.collections.ts
+++ b/src/server.collections.ts
@@ -42,7 +42,7 @@ options: GlobalOptions = getOptions()): McpCollectionCreator => () => {
* Proxy a collection creator with a deferred task wrapper.
*
* @param {McpCollectionCreator} creator - Original creator.
- * @param {CollectionRunSchedule} runSchedule - Schedule config sourced from the collection's
+ * @param {NonNullable['runSchedule']} runSchedule - Schedule config sourced from the collection's
* `_config.runSchedule`. Provides `cancelMs` and `intervalMs` used to build {@link deferTask}.
* @param {GlobalOptions} options - Global options.
* @returns {McpCollectionCreator} The proxied creator function.