From 0a03670cde3a231c5eb82800da9748dab8346c21 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:18:32 -0700 Subject: [PATCH 1/3] fix(chunkers): preserve FAQ prose in docs chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cleanContent deleted every FAQ section from the embedding index: the multiline tag strip swallows an entire block (it matches from ", often inside an answer string), and the brace strip eats any surviving { question, answer } items — 637 Q&As across the docs never reached search, with mangled JSX fragments embedded in their place. Consume FAQ blocks whole before the tag strip and emit their question/answer text as plain prose, escape-aware so braces and quotes inside answers survive. Tag and brace stripping are otherwise unchanged — a corpus survey showed FAQ props are the only place real page prose lives inside JSX syntax on searchable pages. Co-Authored-By: Claude Fable 5 --- apps/sim/lib/chunkers/docs-chunker.test.ts | 105 +++++++++++++++++++++ apps/sim/lib/chunkers/docs-chunker.ts | 33 +++++++ 2 files changed, 138 insertions(+) create mode 100644 apps/sim/lib/chunkers/docs-chunker.test.ts diff --git a/apps/sim/lib/chunkers/docs-chunker.test.ts b/apps/sim/lib/chunkers/docs-chunker.test.ts new file mode 100644 index 00000000000..f0a67329e2f --- /dev/null +++ b/apps/sim/lib/chunkers/docs-chunker.test.ts @@ -0,0 +1,105 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/knowledge/embeddings', () => ({ + generateEmbeddings: vi.fn(async () => ({ embeddings: [] })), + getConfiguredEmbeddingModel: vi.fn(() => 'test-model'), +})) + +import { DocsChunker } from '@/lib/chunkers/docs-chunker' + +function cleanContent(content: string): string { + const chunker = new DocsChunker() + return (chunker as unknown as { cleanContent(content: string): string }).cleanContent.call( + chunker, + content + ) +} + +describe('cleanContent FAQ extraction', () => { + it('keeps FAQ question/answer prose that the tag and brace strips would otherwise delete', () => { + const cleaned = cleanContent( + [ + 'Some intro prose.', + '', + 'import { FAQ } from "@/components/ui/faq"', + '', + '', + ].join('\n') + ) + + expect(cleaned).toContain('What is the maximum file size for uploads?') + expect(cleaned).toContain('20 MB') + expect(cleaned).toContain('standardized UserFile objects') + expect(cleaned).toContain('Some intro prose.') + expect(cleaned).not.toContain('items=') + expect(cleaned).not.toContain('question:') + }) + + it('survives braces and angle-bracket tokens inside answer strings', () => { + const cleaned = cleanContent( + [ + ') and the block extracts what it needs." },', + ']} />', + ].join('\n') + ) + + // Brace placeholders keep their token text; the wrapper chars are dropped + // so the later brace strip cannot punch holes in the sentence. + expect(cleaned).toContain("'data:mime;base64,data'") + // Angle brackets are dropped so the tag strip cannot re-eat the sentence. + expect(cleaned).toContain('(e.g., gmail.attachments[0]) and the block extracts') + }) + + it('extracts items formatted across multiple lines', () => { + const cleaned = cleanContent( + [ + '', + ].join('\n') + ) + + expect(cleaned).toContain('Is SSO supported?') + expect(cleaned).toContain('Yes, on enterprise plans.') + }) + + it('unescapes escaped quotes in extracted strings', () => { + const cleaned = cleanContent( + '' + ) + + expect(cleaned).toContain('What does "draft" mean?') + }) +}) + +describe('cleanContent scaffolding strips', () => { + it('still strips imports, exports, comments, and code-ish brace expressions', () => { + const cleaned = cleanContent( + [ + 'import { Callout } from "fumadocs-ui/components/callout"', + 'export const dynamic = "force-static"', + '{/* editorial note */}', + 'Visible prose {props.title} continues here.', + 'Inside text stays', + ].join('\n') + ) + + expect(cleaned).not.toContain('import') + expect(cleaned).not.toContain('force-static') + expect(cleaned).not.toContain('editorial note') + expect(cleaned).not.toContain('props.title') + expect(cleaned).toContain('Visible prose') + expect(cleaned).toContain('Inside text stays') + }) +}) diff --git a/apps/sim/lib/chunkers/docs-chunker.ts b/apps/sim/lib/chunkers/docs-chunker.ts index 26b1fe449a8..d0f86fc05b9 100644 --- a/apps/sim/lib/chunkers/docs-chunker.ts +++ b/apps/sim/lib/chunkers/docs-chunker.ts @@ -21,6 +21,38 @@ interface Frontmatter { const logger = createLogger('DocsChunker') +/** + * One `{ question: "...", answer: "..." }` FAQ item. Quoted strings are + * consumed escape-aware, so braces or quotes inside an answer never end a + * match early. + */ +const FAQ_ITEM_PATTERN = + /\{\s*question:\s*"((?:[^"\\]|\\.)*)"\s*,\s*answer:\s*"((?:[^"\\]|\\.)*)"\s*\}/g + +function unescapeJsxString(value: string): string { + return value.replace(/\\(.)/g, (_, char: string) => + char === 'n' ? '\n' : char === 't' ? '\t' : char + ) +} + +/** + * Emit an FAQ block's question/answer strings as plain prose lines. Must run + * BEFORE the tag strip: a `` until the + * closing `]} />`, so the multiline tag regex would otherwise swallow the + * items whole (ending at the first `>` inside an answer). Angle brackets and + * braces around inline tokens (``, `data:{mime}`) are + * dropped so the later tag and brace strips cannot re-consume the emitted + * text. + */ +function extractFaqProse(items: string): string { + const lines: string[] = [] + for (const match of items.matchAll(FAQ_ITEM_PATTERN)) { + lines.push(unescapeJsxString(match[1]), unescapeJsxString(match[2])) + } + if (lines.length === 0) return ' ' + return `\n${lines.join('\n').replace(/[<>{}]/g, '')}\n` +} + export class DocsChunker { private readonly textChunker: TextChunker private readonly baseUrl: string @@ -216,6 +248,7 @@ export class DocsChunker { .replace(/\r/g, '\n') .replace(/^import\s+.*$/gm, '') .replace(/^export\s+.*$/gm, '') + .replace(//g, (_m, items: string) => extractFaqProse(items)) .replace(/<\/?[a-zA-Z][^>]*>/g, ' ') .replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ') .replace(/\{[^{}]*\}/g, ' ') From e814d461f3d1240b0219638634de25bf5c43ded5 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:35:50 -0700 Subject: [PATCH 2/3] fix(chunkers): accept single-quoted FAQ items with trailing commas session-policies.mdx and verified-domains.mdx write FAQ items with single-quoted multiline values and trailing commas; the double-quote-only item pattern matched nothing there, so the component consumer replaced those whole FAQ blocks with a space. Capture either quote style escape-aware (quotes of the other style inside a value are fine) and allow the trailing comma; captured values keep their quotes and are unquoted before unescaping. Co-Authored-By: Claude Fable 5 --- apps/sim/lib/chunkers/docs-chunker.test.ts | 26 ++++++++++++++++++++++ apps/sim/lib/chunkers/docs-chunker.ts | 17 +++++++++----- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/chunkers/docs-chunker.test.ts b/apps/sim/lib/chunkers/docs-chunker.test.ts index f0a67329e2f..fb299116ac4 100644 --- a/apps/sim/lib/chunkers/docs-chunker.test.ts +++ b/apps/sim/lib/chunkers/docs-chunker.test.ts @@ -74,6 +74,32 @@ describe('cleanContent FAQ extraction', () => { expect(cleaned).toContain('Yes, on enterprise plans.') }) + it('extracts single-quoted multiline items with trailing commas (session-policies shape)', () => { + const cleaned = cleanContent( + [ + '', + ].join('\n') + ) + + expect(cleaned).toContain('Do session policies apply to SSO sign-ins?') + expect(cleaned).toContain('Yes. Sessions created through SSO follow the same limits.') + expect(cleaned).toContain('Does "Sign out all members" affect API keys?') + expect(cleaned).toContain('No. API keys are unaffected.') + expect(cleaned).not.toContain('items=') + }) + it('unescapes escaped quotes in extracted strings', () => { const cleaned = cleanContent( '' diff --git a/apps/sim/lib/chunkers/docs-chunker.ts b/apps/sim/lib/chunkers/docs-chunker.ts index d0f86fc05b9..3bdcb4a1c23 100644 --- a/apps/sim/lib/chunkers/docs-chunker.ts +++ b/apps/sim/lib/chunkers/docs-chunker.ts @@ -22,12 +22,19 @@ interface Frontmatter { const logger = createLogger('DocsChunker') /** - * One `{ question: "...", answer: "..." }` FAQ item. Quoted strings are - * consumed escape-aware, so braces or quotes inside an answer never end a - * match early. + * One `{ question: "...", answer: "..." }` FAQ item, in either quote style and + * with an optional trailing comma (`session-policies.mdx` uses single-quoted + * multiline items). Each captured value keeps its surrounding quotes — the + * quoted strings are consumed escape-aware per style, so quotes of the other + * style, braces, or escapes inside an answer never end a match early. */ const FAQ_ITEM_PATTERN = - /\{\s*question:\s*"((?:[^"\\]|\\.)*)"\s*,\s*answer:\s*"((?:[^"\\]|\\.)*)"\s*\}/g + /\{\s*question:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*,\s*answer:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*,?\s*\}/g + +/** Strip a captured value's surrounding quotes (either style), then unescape. */ +function unquoteJsxString(value: string): string { + return unescapeJsxString(value.slice(1, -1)) +} function unescapeJsxString(value: string): string { return value.replace(/\\(.)/g, (_, char: string) => @@ -47,7 +54,7 @@ function unescapeJsxString(value: string): string { function extractFaqProse(items: string): string { const lines: string[] = [] for (const match of items.matchAll(FAQ_ITEM_PATTERN)) { - lines.push(unescapeJsxString(match[1]), unescapeJsxString(match[2])) + lines.push(unquoteJsxString(match[1]), unquoteJsxString(match[2])) } if (lines.length === 0) return ' ' return `\n${lines.join('\n').replace(/[<>{}]/g, '')}\n` From b71444b7bf26d6c5459ec7fbda5c65cf9b2ec91e Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:59:18 -0700 Subject: [PATCH 3/3] style(chunkers): wrap the FAQ replace call for biome Co-Authored-By: Claude Fable 5 --- apps/sim/lib/chunkers/docs-chunker.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/chunkers/docs-chunker.ts b/apps/sim/lib/chunkers/docs-chunker.ts index 3bdcb4a1c23..3d3af5d8613 100644 --- a/apps/sim/lib/chunkers/docs-chunker.ts +++ b/apps/sim/lib/chunkers/docs-chunker.ts @@ -255,7 +255,9 @@ export class DocsChunker { .replace(/\r/g, '\n') .replace(/^import\s+.*$/gm, '') .replace(/^export\s+.*$/gm, '') - .replace(//g, (_m, items: string) => extractFaqProse(items)) + .replace(//g, (_m, items: string) => + extractFaqProse(items) + ) .replace(/<\/?[a-zA-Z][^>]*>/g, ' ') .replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ') .replace(/\{[^{}]*\}/g, ' ')