From c3c31f656957e3f2b14201746d0a52a46b06adb5 Mon Sep 17 00:00:00 2001 From: Manus AI Date: Sat, 22 Aug 2026 12:41:14 +0000 Subject: [PATCH] feat(parser): Integrate linkify into parser Use the shared linkify package for plain-text URL detection while retaining the historical matcher as a compatibility fallback. Build linkify before parser consumers in CI and Docker. --- .github/workflows/ci.yml | 4 ++ Dockerfile | 3 ++ packages/markdown-parser/package.json | 3 ++ packages/markdown-parser/src/core/parser.ts | 51 ++++++++------------- packages/markdown-parser/src/index.spec.ts | 10 ++++ packages/markdown-parser/src/index.ts | 1 + packages/markdown-parser/src/types.ts | 4 ++ pnpm-lock.yaml | 4 ++ 8 files changed, 47 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 753516c..bfc3175 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: # Only trigger when relevant code changes to save resources paths: - '.github/workflows/ci.yml' + - 'packages/linkify/**' - 'packages/markdown-parser/**' - 'packages/markdown-parser-lit/**' - 'packages/markdown-parser-vue/**' @@ -18,6 +19,7 @@ on: pull_request: paths: - '.github/workflows/ci.yml' + - 'packages/linkify/**' - 'packages/markdown-parser/**' - 'packages/markdown-parser-lit/**' - 'packages/markdown-parser-vue/**' @@ -53,6 +55,8 @@ jobs: cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts + - name: Build linkify package + run: pnpm --filter @fuyeor/linkify build - name: Typecheck parser package run: pnpm --filter @fuyeor/markdown-parser typecheck - name: Run parser unit tests diff --git a/Dockerfile b/Dockerfile index d9352d3..0d06c37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ WORKDIR /app # prepare depends COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ # copy shared packages +COPY packages/linkify/package.json packages/linkify/ COPY packages/markdown-parser/package.json packages/markdown-parser/ COPY packages/markdown-parser-lit/package.json packages/markdown-parser-lit/ COPY packages/playground/package.json packages/playground/ @@ -23,6 +24,8 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile # copy source code COPY . . +# build linkify before parser consumers resolve its package exports +RUN pnpm --filter @fuyeor/linkify build # build parser package before workspace consumers resolve its package exports RUN pnpm --filter @fuyeor/markdown-parser build diff --git a/packages/markdown-parser/package.json b/packages/markdown-parser/package.json index 035cbe7..89d2d0d 100644 --- a/packages/markdown-parser/package.json +++ b/packages/markdown-parser/package.json @@ -18,6 +18,9 @@ "exports": { ".": "./src/index.ts" }, + "dependencies": { + "@fuyeor/linkify": "workspace:*" + }, "devDependencies": { "@types/node": "^25.6.0", "typescript": "^6.0.3", diff --git a/packages/markdown-parser/src/core/parser.ts b/packages/markdown-parser/src/core/parser.ts index b4b0b61..d244548 100644 --- a/packages/markdown-parser/src/core/parser.ts +++ b/packages/markdown-parser/src/core/parser.ts @@ -1,17 +1,24 @@ // @fuyeor/markdown-parser/src/core/parser.ts import { BlockState, InlineState } from './state'; +import { linkify } from '@fuyeor/linkify'; import type { ASTNode, BlockRule, InlineRule, + Linkifier, MarkdownPlugin, MarkdownParserOptions, ParserContext, } from '#/types'; import { isSafeLinkUrl } from '#/core/url'; -const LINKIFY_REGEX = - /(https?:\/\/[^\s]+|(? { + if (!LINKIFY_CANDIDATE_REGEX.test(text)) return []; + return linkify(text); +} export class MarkdownParser { #blockRuleMap = new Map(); @@ -19,6 +26,7 @@ export class MarkdownParser { #idSequence = 0; #isPreflight = false; readonly #maxNestingDepth: number; + readonly #linkifier: Linkifier; constructor(options: MarkdownParserOptions = {}) { const maxNestingDepth = options.maxNestingDepth ?? 64; @@ -26,6 +34,7 @@ export class MarkdownParser { throw new RangeError('maxNestingDepth must be a positive integer'); this.#maxNestingDepth = maxNestingDepth; + this.#linkifier = options.linkifier ?? linkifyWithCandidateCheck; } // construct a context for recursive rule calls @@ -227,44 +236,20 @@ export class MarkdownParser { const textBuffer = state.content.slice(textStart, endPos); let lastIdx = 0; - for (const match of textBuffer.matchAll(LINKIFY_REGEX)) { - let urlStr = match[0]; - let matchIdx = match.index!; - - // remove the punctuation marks at the end. - const punctuation = '.,:;?!'; - while ( - urlStr.length > 0 && - punctuation.includes(urlStr[urlStr.length - 1]) - ) { - urlStr = urlStr.slice(0, -1); - } - - // handle bracket matching - // If a link ends with `)` and the inner brackets are unbalanced, then strip `)` - if (urlStr.endsWith(')')) { - const openCount = (urlStr.match(/\(/g) || []).length; - const closeCount = (urlStr.match(/\)/g) || []).length; - if (closeCount > openCount) urlStr = urlStr.slice(0, -1); - } - - const fullUrl = urlStr.startsWith('http') - ? urlStr - : `https://${urlStr}`; - - if (isSafeLinkUrl(fullUrl)) { - if (matchIdx > lastIdx) { + for (const match of this.#linkifier(textBuffer)) { + if (isSafeLinkUrl(match.url)) { + if (match.index > lastIdx) { nodes.push({ type: 'text', - content: textBuffer.slice(lastIdx, matchIdx), + content: textBuffer.slice(lastIdx, match.index), }); } nodes.push({ type: 'link', - url: fullUrl, - children: [{ type: 'text', content: urlStr }], + url: match.url, + children: [{ type: 'text', content: match.text }], }); - lastIdx = matchIdx + urlStr.length; + lastIdx = match.lastIndex; } } diff --git a/packages/markdown-parser/src/index.spec.ts b/packages/markdown-parser/src/index.spec.ts index 1f828f2..8ce7866 100644 --- a/packages/markdown-parser/src/index.spec.ts +++ b/packages/markdown-parser/src/index.spec.ts @@ -56,6 +56,16 @@ describe('test @fuyeor/markdown-parser', () => { expect(children[3].url).toBe('https://fuyeor.com'); }); + it('uses linkify for internationalized fuzzy domains', () => { + const ast = createFuyeorMarkdownParser()('Visit fuyeor.xn--p1ai'); + const link = ast[0].children?.find((node) => node.type === 'link'); + + expect(link).toMatchObject({ + url: 'https://fuyeor.рф', + children: [{ type: 'text', content: 'fuyeor.рф' }], + }); + }); + it('parse table', () => { const content = ` | title 1 | title 2 | diff --git a/packages/markdown-parser/src/index.ts b/packages/markdown-parser/src/index.ts index 2451ef8..aaa247b 100644 --- a/packages/markdown-parser/src/index.ts +++ b/packages/markdown-parser/src/index.ts @@ -27,6 +27,7 @@ export type { ParserContext, BlockRule, InlineRule, + Linkifier, MarkdownParserOptions, MarkdownPlugin, } from './types'; diff --git a/packages/markdown-parser/src/types.ts b/packages/markdown-parser/src/types.ts index 26ca104..460d45d 100644 --- a/packages/markdown-parser/src/types.ts +++ b/packages/markdown-parser/src/types.ts @@ -1,5 +1,6 @@ // @fuyeor/markdown-parser/src/types.ts +import type { LinkMatch } from '@fuyeor/linkify'; import type { MarkdownParser } from './core/parser'; import type { BlockState, InlineState } from './core/state'; @@ -73,8 +74,11 @@ export interface InlineRule { ) => { node: ASTNode; consumedChars: number } | null; } +export type Linkifier = (text: string) => readonly LinkMatch[]; + export interface MarkdownParserOptions { maxNestingDepth?: number; + linkifier?: Linkifier; } export type MarkdownPlugin = (parser: MarkdownParser) => void; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0835787..4449ed7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,10 @@ importers: version: 4.1.4(@types/node@25.6.0)(@vitest/coverage-v8@4.1.4)(jsdom@29.0.2)(vite@8.0.9(@types/node@25.6.0)(esbuild@0.27.7)(tsx@4.21.0)) packages/markdown-parser: + dependencies: + '@fuyeor/linkify': + specifier: workspace:* + version: link:../linkify devDependencies: '@types/node': specifier: ^25.6.0