Skip to content

Commit 71e848a

Browse files
waleedlatif1waleedlatif1
authored
fix(chat): preserve literal code around source chips (#7456)
* fix(chat): preserve literal code around source chips * fix(chat): scope delimiter pairing to paragraphs * fix(chat): preserve tilde-fenced citations
1 parent 44a1612 commit 71e848a

3 files changed

Lines changed: 235 additions & 51 deletions

File tree

‎apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,104 @@ import { sanitizeChatDisplayContent } from '@/app/workspace/[workspaceId]/home/c
66
import { scalingRatioOver4x } from '@/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers'
77

88
describe('sanitizeChatDisplayContent', () => {
9+
it.each(['source', 'workspace_resource'])(
10+
'unwraps %s JSON that mentions the other chip tag',
11+
(name) => {
12+
const otherTag = name === 'source' ? 'workspace_resource' : 'source'
13+
const tag = `<${name}>${JSON.stringify({ title: `Use <${otherTag}>` })}</${name}>`
14+
15+
expect(sanitizeChatDisplayContent(`\`${tag}\``)).toBe(tag)
16+
}
17+
)
18+
19+
it.each([2, 3, 4])('preserves a %i-backtick code span containing a chip', (length) => {
20+
const delimiter = '`'.repeat(length)
21+
const tag = '<source>{"url":"https://example.com","title":"Use `config`"}</source>'
22+
const content = `${delimiter}${tag}${delimiter}`
23+
24+
expect(sanitizeChatDisplayContent(content)).toBe(content)
25+
expect(sanitizeChatDisplayContent(`${delimiter}json\n\`${tag}\`\n${delimiter}`)).toBe(
26+
`${delimiter}json\n\`${tag}\`\n${delimiter}`
27+
)
28+
expect(sanitizeChatDisplayContent(`${content} then \`${tag}\``)).toBe(`${content} then ${tag}`)
29+
})
30+
31+
it('does not let an unmatched backtick run suppress later citations', () => {
32+
const prefix = 'Use `` for two backticks.\n'
33+
const tag = '<source>{"url":"https://example.com"}</source>'
34+
35+
expect(sanitizeChatDisplayContent(`${prefix}\`${tag}\``)).toBe(`${prefix}${tag}`)
36+
})
37+
38+
it.each(['\n\n', '\r\n\r\n', '\n \t\n'])(
39+
'does not pair prose runs across paragraph break %j',
40+
(separator) => {
41+
const tag = '<source>{"url":"https://example.com"}</source>'
42+
const before = `Use \`\` as a delimiter.${separator}`
43+
const after = `${separator}Another \`\` marker.`
44+
45+
expect(sanitizeChatDisplayContent(`${before}\`${tag}\`${after}`)).toBe(
46+
`${before}${tag}${after}`
47+
)
48+
}
49+
)
50+
51+
it('preserves matched multi-backtick spans across a soft line break', () => {
52+
const content = '``Literal\n`<source>{"url":"https://example.com"}</source>`\nexample``'
53+
54+
expect(sanitizeChatDisplayContent(content)).toBe(content)
55+
})
56+
57+
it('does not treat blank lines inside chip JSON as paragraph breaks', () => {
58+
const tag = '<source>{\n\n"url":"https://example.com",\n\n"title":"Use `code`"\n}</source>'
59+
60+
expect(sanitizeChatDisplayContent(`\`${tag}\``)).toBe(tag)
61+
})
62+
63+
it.each(['```', '~~~'])(
64+
'preserves a %s fence closed by a longer run and unwraps citations after it',
65+
(fence) => {
66+
const tag = '<source>{"url":"https://example.com"}</source>'
67+
const block = `${fence}json\n\`${tag}\`\n${fence}${fence[0]}\n`
68+
69+
expect(sanitizeChatDisplayContent(`${block}\`${tag}\``)).toBe(`${block}${tag}`)
70+
}
71+
)
72+
73+
it.each(['```', '~~~'])('leaves an unclosed %s streaming fence literal', (fence) => {
74+
const content = `${fence}json\n\`<source>{"url":"https://example.com"}</source>\``
75+
76+
expect(sanitizeChatDisplayContent(content)).toBe(content)
77+
})
78+
79+
it.each(['source', 'workspace_resource'])(
80+
'preserves tilde-fenced %s chips with backticks in the info string',
81+
(name) => {
82+
const tag = `<${name}>{"title":"Example"}</${name}>`
83+
const block = `~~~example \`code\`\n\`${tag}\`\n~~~\n`
84+
85+
expect(sanitizeChatDisplayContent(`${block}\`${tag}\``)).toBe(`${block}${tag}`)
86+
}
87+
)
88+
89+
it.each(['```', '~~~'])(
90+
'does not close a %s fence with a different character or a shorter run',
91+
(fence) => {
92+
const tag = '<source>{"url":"https://example.com"}</source>'
93+
const otherFence = fence === '```' ? '~~~~' : '````'
94+
const block = `${fence}${fence[0]}\n${otherFence}\n\`${tag}\`\n${fence}\n\`${tag}\`\n${fence}${fence[0]}\n`
95+
96+
expect(sanitizeChatDisplayContent(`${block}\`${tag}\``)).toBe(`${block}${tag}`)
97+
}
98+
)
99+
100+
it('does not open a backtick fence with backticks in its info string', () => {
101+
const prefix = '```example `code`\n\n'
102+
const tag = '<source>{"url":"https://example.com"}</source>'
103+
104+
expect(sanitizeChatDisplayContent(`${prefix}\`${tag}\``)).toBe(`${prefix}${tag}`)
105+
})
106+
9107
it('unwraps workspace resource tags from inline code spans', () => {
10108
const content =
11109
'`I updated <workspace_resource>{"type":"workflow","id":"wf-1","title":"Workflow"}</workspace_resource>.`'
@@ -159,4 +257,24 @@ describe('sanitizeChatDisplayContent', () => {
159257
'<workspace_resource>{"type":"file","path":"a.md","title":"a"}</workspace_resource> done'
160258
)
161259
})
260+
261+
it.each(['source', 'workspace_resource'])(
262+
'stays linear on repeated %s tags with unterminated JSON strings',
263+
(name) => {
264+
expect(
265+
scalingRatioOver4x(sanitizeChatDisplayContent, (times) =>
266+
`<${name}>{${String.fromCharCode(92, 34)}`.repeat(times)
267+
)
268+
).toBeLessThan(8)
269+
}
270+
)
271+
272+
it.each(['<source>"', '<source>{"key":"'])(
273+
'stays linear on repeated quoted payload prefix %s',
274+
(prefix) => {
275+
expect(
276+
scalingRatioOver4x(sanitizeChatDisplayContent, (times) => prefix.repeat(times))
277+
).toBeLessThan(8)
278+
}
279+
)
162280
})

‎apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts‎

Lines changed: 110 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,129 @@
11
const HIDDEN_INLINE_REFERENCE_PATTERN =
22
/`[^`\n]*(?:internal\/tool-results\/|internal\/blocktips\/|components\/integrations\/[^`\n]*README)[^`\n]*`/g
33

4-
/** JSON strings own their escaped quotes, backticks, and any quoted tag markers. */
5-
const JSON_STRING_SOURCE = String.raw`"(?:[^"\\\r\n]|\\[^\r\n])*"`
4+
/** JSON strings own their escaped quotes, backticks, and quoted tag markers. */
5+
const JSON_STRING_SOURCE = '"(?:\\\\(?:["\\\\/bfnrt]|u[0-9a-fA-F]{4})|[^"\\\\\\r\\n])*"'
66

7-
/**
8-
* Complete chip tags consume JSON strings atomically. Outside strings, a new
9-
* opener or backtick ends the candidate, so prose mentions cannot join into a
10-
* tag and repeated unclosed openers cannot repeatedly scan the same suffix.
11-
*/
12-
const COMPLETE_TAG_SOURCE = `<(?<chipTag>workspace_resource|source)>\\s*\\{(?:${JSON_STRING_SOURCE}|[^"\`<])*?\\}\\s*</\\k<chipTag>>`
7+
/** Unquoted openers, backticks, and invalid backslashes bound failed payload scans. */
8+
const COMPLETE_TAG_SOURCE = `<(?<chipTag>workspace_resource|source)>\\s*\\{(?:${JSON_STRING_SOURCE}|[^"\`<\\\\])*?\\}\\s*</\\k<chipTag>>`
139

14-
const CHIP_OR_CODE_DELIMITER = new RegExp(`${COMPLETE_TAG_SOURCE}|\`|\n`, 'g')
10+
const INLINE_CHIP_OR_DELIMITER = new RegExp(`${COMPLETE_TAG_SOURCE}|\`+|\\n`, 'g')
11+
const CHIP_OR_PARAGRAPH_BREAK = new RegExp(`${COMPLETE_TAG_SOURCE}|\\n[\\t \\r]*\\n`, 'g')
1512

16-
/**
17-
* Pair Markdown delimiters outside chip payloads in one forward pass. A pair
18-
* containing a chip is unwrapped; a lone delimiter is removed only when flush
19-
* against a chip. Neighbouring code spans and multiline fences keep their pairs.
20-
*/
21-
export function sanitizeChatDisplayContent(content: string): string {
22-
const removedDelimiters: number[] = []
23-
let openingTick = -1
24-
let containsChip = false
25-
let adjacentToChip = false
26-
let lastChipEnd = -1
13+
interface OpenCodeSpan {
14+
index: number
15+
containsChip: boolean
16+
touchesChip: boolean
17+
}
2718

28-
for (const match of content.matchAll(CHIP_OR_CODE_DELIMITER)) {
29-
const index = match.index
30-
if (match.groups?.chipTag) {
31-
if (openingTick !== -1) {
32-
containsChip = true
33-
adjacentToChip ||= index === openingTick + 1
34-
}
35-
lastChipEnd = index + match[0].length
36-
continue
19+
/** Only matched multi-backtick runs are code; an unmatched run remains ordinary prose. */
20+
function unwrapInlineParagraph(content: string): string {
21+
const remainingRuns = new Map<number, number>()
22+
for (const [value] of content.matchAll(INLINE_CHIP_OR_DELIMITER)) {
23+
if (value.startsWith('`') && value.length > 1) {
24+
remainingRuns.set(value.length, (remainingRuns.get(value.length) ?? 0) + 1)
3725
}
26+
}
27+
const removedDelimiters: number[] = []
28+
let openSpan: OpenCodeSpan | null = null
29+
let previousChipEnd = -1
30+
let protectedRunLength: number | null = null
3831

39-
if (match[0] === '\n') {
40-
if (openingTick !== -1 && adjacentToChip) removedDelimiters.push(openingTick)
41-
openingTick = -1
42-
lastChipEnd = -1
43-
continue
44-
}
32+
const finishLine = () => {
33+
if (openSpan?.touchesChip) removedDelimiters.push(openSpan.index)
34+
openSpan = null
35+
}
4536

46-
if (openingTick === -1) {
47-
openingTick = index
48-
containsChip = false
49-
adjacentToChip = lastChipEnd === index
37+
for (const token of content.matchAll(INLINE_CHIP_OR_DELIMITER)) {
38+
const [value] = token
39+
const index = token.index
40+
if (value === '\n') {
41+
finishLine()
42+
previousChipEnd = -1
43+
} else if (value.startsWith('`')) {
44+
if (value.length > 1) {
45+
remainingRuns.set(value.length, (remainingRuns.get(value.length) ?? 1) - 1)
46+
}
47+
if (protectedRunLength !== null) {
48+
if (value.length === protectedRunLength) protectedRunLength = null
49+
continue
50+
}
51+
if (value.length > 1) {
52+
if (!openSpan && remainingRuns.get(value.length)) protectedRunLength = value.length
53+
continue
54+
}
55+
if (openSpan) {
56+
if (openSpan.containsChip) removedDelimiters.push(openSpan.index, index)
57+
openSpan = null
58+
} else {
59+
openSpan = { index, containsChip: false, touchesChip: previousChipEnd === index }
60+
}
5061
} else {
51-
if (containsChip) removedDelimiters.push(openingTick, index)
52-
openingTick = -1
62+
if (openSpan) {
63+
openSpan.containsChip = true
64+
openSpan.touchesChip ||= index === openSpan.index + 1
65+
}
66+
previousChipEnd = index + value.length
5367
}
5468
}
55-
56-
if (openingTick !== -1 && adjacentToChip) removedDelimiters.push(openingTick)
69+
finishLine()
5770

5871
const parts: string[] = []
59-
let start = 0
72+
let cursor = 0
6073
for (const index of removedDelimiters) {
61-
parts.push(content.slice(start, index))
62-
start = index + 1
74+
parts.push(content.slice(cursor, index))
75+
cursor = index + 1
76+
}
77+
parts.push(content.slice(cursor))
78+
return parts.join('')
79+
}
80+
81+
/** Paragraph breaks end inline spans, but blank lines inside chip JSON belong to the payload. */
82+
function unwrapInlineChips(content: string): string {
83+
const parts: string[] = []
84+
let cursor = 0
85+
86+
for (const match of content.matchAll(CHIP_OR_PARAGRAPH_BREAK)) {
87+
if (!match[0].startsWith('\n')) continue
88+
parts.push(unwrapInlineParagraph(content.slice(cursor, match.index)), match[0])
89+
cursor = match.index + match[0].length
90+
}
91+
parts.push(unwrapInlineParagraph(content.slice(cursor)))
92+
return parts.join('')
93+
}
94+
95+
/** Fenced blocks are literal, including unclosed streaming fences and longer closing runs. */
96+
export function sanitizeChatDisplayContent(content: string): string {
97+
const parts: string[] = []
98+
let cursor = 0
99+
let fenceStart: number | null = null
100+
let fence = ''
101+
102+
for (const line of content.matchAll(/^ {0,3}(`{3,}|~{3,})([^\n]*)(?:\n|$)/gm)) {
103+
const [, delimiter, info] = line
104+
if (fenceStart === null) {
105+
if (delimiter[0] === '`' && info.includes('`')) continue
106+
fenceStart = line.index
107+
fence = delimiter
108+
} else if (
109+
delimiter[0] === fence[0] &&
110+
delimiter.length >= fence.length &&
111+
/^[\t \r]*$/.test(info)
112+
) {
113+
const end = line.index + line[0].length
114+
parts.push(
115+
unwrapInlineChips(content.slice(cursor, fenceStart)),
116+
content.slice(fenceStart, end)
117+
)
118+
cursor = end
119+
fenceStart = null
120+
}
121+
}
122+
123+
if (fenceStart === null) {
124+
parts.push(unwrapInlineChips(content.slice(cursor)))
125+
} else {
126+
parts.push(unwrapInlineChips(content.slice(cursor, fenceStart)), content.slice(fenceStart))
63127
}
64-
parts.push(content.slice(start))
65128
return parts.join('').replace(HIDDEN_INLINE_REFERENCE_PATTERN, '')
66129
}

‎apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,16 @@ function fastest(run: (content: string) => void, content: string): number {
2727
* through at the single size it happens to sample. Quadratic costs ~16x for 4x
2828
* the input; linear costs ~4x.
2929
*/
30-
export function scalingRatioOver4x(run: (content: string) => void): number {
30+
export function scalingRatioOver4x(
31+
run: (content: string) => void,
32+
buildContent: (times: number) => string = buildRepeatedTagMentions
33+
): number {
3134
// Warm up first — the JIT would otherwise charge the whole compile to the
3235
// small sample and flatter the ratio.
33-
fastest(run, buildRepeatedTagMentions(2_000))
36+
fastest(run, buildContent(2_000))
3437

35-
const small = fastest(run, buildRepeatedTagMentions(2_000))
36-
const large = fastest(run, buildRepeatedTagMentions(8_000))
38+
const small = fastest(run, buildContent(2_000))
39+
const large = fastest(run, buildContent(8_000))
3740

3841
return large / small
3942
}

0 commit comments

Comments
 (0)