diff --git a/apps/sim/lib/chunkers/docs-chunker.test.ts b/apps/sim/lib/chunkers/docs-chunker.test.ts new file mode 100644 index 00000000000..fb299116ac4 --- /dev/null +++ b/apps/sim/lib/chunkers/docs-chunker.test.ts @@ -0,0 +1,131 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/knowledge/embeddings', () => ({ + generateEmbeddings: vi.fn(async () => ({ embeddings: [] })), + getConfiguredEmbeddingModel: vi.fn(() => 'test-model'), +})) + +import { DocsChunker } from '@/lib/chunkers/docs-chunker' + +function cleanContent(content: string): string { + const chunker = new DocsChunker() + return (chunker as unknown as { cleanContent(content: string): string }).cleanContent.call( + chunker, + content + ) +} + +describe('cleanContent FAQ extraction', () => { + it('keeps FAQ question/answer prose that the tag and brace strips would otherwise delete', () => { + const cleaned = cleanContent( + [ + 'Some intro prose.', + '', + 'import { FAQ } from "@/components/ui/faq"', + '', + '', + ].join('\n') + ) + + expect(cleaned).toContain('What is the maximum file size for uploads?') + expect(cleaned).toContain('20 MB') + expect(cleaned).toContain('standardized UserFile objects') + expect(cleaned).toContain('Some intro prose.') + expect(cleaned).not.toContain('items=') + expect(cleaned).not.toContain('question:') + }) + + it('survives braces and angle-bracket tokens inside answer strings', () => { + const cleaned = cleanContent( + [ + ') and the block extracts what it needs." },', + ']} />', + ].join('\n') + ) + + // Brace placeholders keep their token text; the wrapper chars are dropped + // so the later brace strip cannot punch holes in the sentence. + expect(cleaned).toContain("'data:mime;base64,data'") + // Angle brackets are dropped so the tag strip cannot re-eat the sentence. + expect(cleaned).toContain('(e.g., gmail.attachments[0]) and the block extracts') + }) + + it('extracts items formatted across multiple lines', () => { + const cleaned = cleanContent( + [ + '', + ].join('\n') + ) + + expect(cleaned).toContain('Is SSO supported?') + expect(cleaned).toContain('Yes, on enterprise plans.') + }) + + it('extracts single-quoted multiline items with trailing commas (session-policies shape)', () => { + const cleaned = cleanContent( + [ + '', + ].join('\n') + ) + + expect(cleaned).toContain('Do session policies apply to SSO sign-ins?') + expect(cleaned).toContain('Yes. Sessions created through SSO follow the same limits.') + expect(cleaned).toContain('Does "Sign out all members" affect API keys?') + expect(cleaned).toContain('No. API keys are unaffected.') + expect(cleaned).not.toContain('items=') + }) + + it('unescapes escaped quotes in extracted strings', () => { + const cleaned = cleanContent( + '' + ) + + expect(cleaned).toContain('What does "draft" mean?') + }) +}) + +describe('cleanContent scaffolding strips', () => { + it('still strips imports, exports, comments, and code-ish brace expressions', () => { + const cleaned = cleanContent( + [ + 'import { Callout } from "fumadocs-ui/components/callout"', + 'export const dynamic = "force-static"', + '{/* editorial note */}', + 'Visible prose {props.title} continues here.', + 'Inside text stays', + ].join('\n') + ) + + expect(cleaned).not.toContain('import') + expect(cleaned).not.toContain('force-static') + expect(cleaned).not.toContain('editorial note') + expect(cleaned).not.toContain('props.title') + expect(cleaned).toContain('Visible prose') + expect(cleaned).toContain('Inside text stays') + }) +}) diff --git a/apps/sim/lib/chunkers/docs-chunker.ts b/apps/sim/lib/chunkers/docs-chunker.ts index 26b1fe449a8..3d3af5d8613 100644 --- a/apps/sim/lib/chunkers/docs-chunker.ts +++ b/apps/sim/lib/chunkers/docs-chunker.ts @@ -21,6 +21,45 @@ interface Frontmatter { const logger = createLogger('DocsChunker') +/** + * One `{ question: "...", answer: "..." }` FAQ item, in either quote style and + * with an optional trailing comma (`session-policies.mdx` uses single-quoted + * multiline items). Each captured value keeps its surrounding quotes — the + * quoted strings are consumed escape-aware per style, so quotes of the other + * style, braces, or escapes inside an answer never end a match early. + */ +const FAQ_ITEM_PATTERN = + /\{\s*question:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*,\s*answer:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*,?\s*\}/g + +/** Strip a captured value's surrounding quotes (either style), then unescape. */ +function unquoteJsxString(value: string): string { + return unescapeJsxString(value.slice(1, -1)) +} + +function unescapeJsxString(value: string): string { + return value.replace(/\\(.)/g, (_, char: string) => + char === 'n' ? '\n' : char === 't' ? '\t' : char + ) +} + +/** + * Emit an FAQ block's question/answer strings as plain prose lines. Must run + * BEFORE the tag strip: a `` until the + * closing `]} />`, so the multiline tag regex would otherwise swallow the + * items whole (ending at the first `>` inside an answer). Angle brackets and + * braces around inline tokens (``, `data:{mime}`) are + * dropped so the later tag and brace strips cannot re-consume the emitted + * text. + */ +function extractFaqProse(items: string): string { + const lines: string[] = [] + for (const match of items.matchAll(FAQ_ITEM_PATTERN)) { + lines.push(unquoteJsxString(match[1]), unquoteJsxString(match[2])) + } + if (lines.length === 0) return ' ' + return `\n${lines.join('\n').replace(/[<>{}]/g, '')}\n` +} + export class DocsChunker { private readonly textChunker: TextChunker private readonly baseUrl: string @@ -216,6 +255,9 @@ export class DocsChunker { .replace(/\r/g, '\n') .replace(/^import\s+.*$/gm, '') .replace(/^export\s+.*$/gm, '') + .replace(//g, (_m, items: string) => + extractFaqProse(items) + ) .replace(/<\/?[a-zA-Z][^>]*>/g, ' ') .replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ') .replace(/\{[^{}]*\}/g, ' ')