diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/features/markdown/logic/markdown.ts | 90 | ||||
| -rw-r--r-- | src/features/markdown/logic/streaming-markdown.test.ts | 323 | ||||
| -rw-r--r-- | src/features/markdown/logic/streaming-markdown.ts | 480 | ||||
| -rw-r--r-- | src/features/markdown/ui/Markdown.svelte | 38 | ||||
| -rw-r--r-- | src/features/markdown/ui/markdown.test.ts | 93 |
5 files changed, 1012 insertions, 12 deletions
diff --git a/src/features/markdown/logic/markdown.ts b/src/features/markdown/logic/markdown.ts index ad8a8bd..626b4da 100644 --- a/src/features/markdown/logic/markdown.ts +++ b/src/features/markdown/logic/markdown.ts @@ -125,19 +125,50 @@ const md = new Marked( /** * While a message is still streaming, balance dangling fences / emphasis so the - * partial text renders cleanly instead of flashing raw markers. + * partial text renders cleanly instead of flashing raw markers. `seed` carries + * the delimiter counts of the ALREADY-SETTLED prefix (see + * `logic/streaming-markdown`): the full-text streaming render appends markers + * at the very END of the whole source, so a tail rendered alone must count the + * prefix's parity to append in exactly the same place (or not) — otherwise a + * settled paragraph with a lone `**` would transiently render differently. */ -function closeOpenDelimiters(src: string): string { +function closeOpenDelimiters( + src: string, + seed: DelimiterCounts = { fences: 0, bold: 0, code: 0 }, +): string { let out = src; - const fenceCount = (out.match(/^```/gm) ?? []).length; + const fenceCount = (out.match(/^```/gm) ?? []).length + seed.fences; if (fenceCount % 2 !== 0) out += "\n```"; - const boldCount = (out.match(/\*\*/g) ?? []).length; + const boldCount = (out.match(/\*\*/g) ?? []).length + seed.bold; if (boldCount % 2 !== 0) out += "**"; - const inlineCode = (out.match(/(?<!`)`(?!`)/g) ?? []).length; + const inlineCode = (out.match(/(?<!`)`(?!`)/g) ?? []).length + seed.code; if (inlineCode % 2 !== 0) out += "`"; return out; } +/** + * Counts of the delimiter constructs {@link closeOpenDelimiters} balances, + * using the SAME regexes — the streaming checkpoint module accumulates them + * over settled blocks and seeds the tail's balance pass with the total, so the + * tail's appended markers land exactly where a whole-text render would put + * them. Pure: `input → output`. + */ +export interface DelimiterCounts { + readonly fences: number; + readonly bold: number; + readonly code: number; +} + +const ZERO_COUNTS: DelimiterCounts = { fences: 0, bold: 0, code: 0 }; + +export function countDelimiters(src: string): DelimiterCounts { + return { + fences: (src.match(/^```/gm) ?? []).length, + bold: (src.match(/\*\*/g) ?? []).length, + code: (src.match(/(?<!`)`(?!`)/g) ?? []).length, + }; +} + // Wrap each fenced code block (`<pre>…</pre>`) in a positioned container with a // copy button. marked emits exactly one `<pre>`/`</pre>` pair per block and // escapes `<`/`>` inside code, so these literal tags only ever delimit blocks. @@ -163,3 +194,52 @@ export function renderMarkdown(text: string, opts?: { streaming?: boolean }): st return ""; } } + +/** + * The streaming TAIL pipeline — today's exact pipeline for the UNSETTLED tail + * of a streaming message (`closeOpenDelimiters` → marked → copy buttons → + * DOMPurify), with the delimiter parity SEEDED from the settled prefix (see + * {@link countDelimiters}). With a zero seed this is byte-identical to + * `renderMarkdown(text, { streaming: true })`, so the checkpoint module's + * degenerate case (nothing settled yet) is exactly today's behavior. + * Returns `""` if parsing ever throws. + */ +export function renderMarkdownTail(text: string, seed: DelimiterCounts = ZERO_COUNTS): string { + const src = closeOpenDelimiters(text, seed); + try { + const raw = md.parse(src) as string; + return DOMPurify.sanitize(addCopyButtons(raw)); + } catch { + return ""; + } +} + +/** + * Escapes code content the way marked does inside `<code>` (only `&`, `<`, + * `>` — quotes stay literal) so the plain open-fence render below is + * character-identical to what the closed-fence pipeline would emit. + */ +function escapeCode(s: string): string { + return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">"); +} + +/** + * Plain (UN-highlighted) rendering of an OPEN fenced code block — the Level-2 + * streaming behavior: while a fence is still open its content is final-format + * escaped `<pre><code>` with NO hljs (hljs state spans lines — an unterminated + * string/block comment would recolor earlier lines, so line-incremental + * highlighting is unsound). The block is highlighted exactly ONCE, when it + * closes and settles into the checkpoint cache. Emits the same copy-button + * wrapper + `hljs language-X` classes as the closed pipeline (modulo color + * spans), so the pop-in on close is "just colour". + */ +export function renderOpenFence(info: string, code: string): string { + const lang = info.trim().split(/\s+/)[0] ?? ""; + const cls = lang === "" ? "hljs" : `hljs language-${escapeHtml(lang)}`; + const html = `<pre><code class="${cls}">${escapeCode(code)}</code></pre>`; + try { + return DOMPurify.sanitize(addCopyButtons(html)); + } catch { + return ""; + } +} diff --git a/src/features/markdown/logic/streaming-markdown.test.ts b/src/features/markdown/logic/streaming-markdown.test.ts new file mode 100644 index 0000000..f810579 --- /dev/null +++ b/src/features/markdown/logic/streaming-markdown.test.ts @@ -0,0 +1,323 @@ +import hljs from "highlight.js/lib/core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { countDelimiters, renderMarkdown, renderMarkdownTail, renderOpenFence } from "./markdown"; +import { advance, createStreamingMarkdown } from "./streaming-markdown"; + +/** + * Test-side normalizer for the equivalence contract. The checkpoint render may + * legitimately differ from today's whole-text streaming render in exactly two + * transient ways while streaming: + * 1. Level 2: an OPEN fence renders plain-escaped (no hljs spans) — unwrapping + * hljs spans on both sides makes them comparable ("just colour" is the + * accepted trade; the closed render is byte-identical). + * 2. Newlines BETWEEN top-level blocks (each rendered unit emits its own + * trailing newline) and marked's trailing newline inside `<code>` before an + * appended closing fence — inter-tag whitespace collapse handles both. + * Text-node content inside `<pre><code>` is untouched (`>…<` never matches + * escaped code content), so code bytes are still compared exactly. + */ +function normalizeHtml(html: string): string { + const host = document.createElement("div"); + host.innerHTML = html; + for (const span of [...host.querySelectorAll("span")]) { + if ([...span.classList].some((c) => c.startsWith("hljs"))) { + span.replaceWith(...span.childNodes); + } + } + return host.innerHTML + .replace(/>\s+</g, "><") + .replace(/\n+(?=<\/code>)/g, "") + .trim(); +} + +describe("streaming-markdown", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("degenerate case (nothing settled) is exactly today's streaming render", () => { + const state = createStreamingMarkdown(); + const text = "Some **bold** and `code` text"; + const { prefixHtml, tailHtml } = advance(state, text); + expect(prefixHtml).toBe(""); + expect(tailHtml).toBe(renderMarkdown(text, { streaming: true })); + }); + + it("settles a paragraph at a blank-line boundary and matches the full render", () => { + const state = createStreamingMarkdown(); + // The settle commits only once the boundary's lookahead line completes. + advance(state, "first para\n\nnext"); + expect(state.prefixHtml).toBe(""); + advance(state, "first para\n\nnext\n"); + // The settle point is the START of the blank run (the separator blanks + // lead the tail, where marked ignores them — the rendered HTML is the + // same either way). + expect(state.prefixHtml).toBe(renderMarkdown("first para\n")); + expect(state.settledOffset).toBe("first para\n".length); + // Equivalence against the whole-text render. + const { prefixHtml, tailHtml } = advance(state, "first para\n\nnext\n"); + expect(normalizeHtml(prefixHtml + tailHtml)).toBe( + normalizeHtml(renderMarkdown("first para\n\nnext\n", { streaming: true })), + ); + }); + + it("matches the full render while inside an open list (no checkpoints — degraded caching, never wrong)", () => { + const state = createStreamingMarkdown(); + const text = "- item a\n\n- item b\n\n- item c"; + let shown = ""; + for (let i = 1; i <= text.length; i++) { + const prefix = text.slice(0, i); + const r = advance(state, prefix); + shown = r.prefixHtml + r.tailHtml; + expect(normalizeHtml(shown)).toBe(normalizeHtml(renderMarkdown(prefix, { streaming: true }))); + } + }); + + it("highlight-exactly-once: zero hljs while a fence is open, one call when it closes", () => { + const spy = vi.spyOn(hljs, "highlight"); + const state = createStreamingMarkdown(); + + advance(state, "intro\n\n```js\nc"); + expect(spy).not.toHaveBeenCalled(); // open fence → plain escaped, no hljs + + advance(state, "intro\n\n```js\nconst x = 1;\n"); + expect(spy).not.toHaveBeenCalled(); // still open + + const closed = advance(state, "intro\n\n```js\nconst x = 1;\n```\n"); + expect(spy).toHaveBeenCalledTimes(1); // highlighted EXACTLY once, on close + expect(closed.prefixHtml).toContain("hljs-keyword"); // the settled fence is colored + // The open-fence tail is gone (nothing but prose after the fence). + expect(closed.tailHtml).toBe(renderMarkdownTail("", state.seed)); + + // Subsequent deltas do NOT re-highlight the settled block. + advance(state, "intro\n\n```js\nconst x = 1;\n```\nafter\n"); + advance(state, "intro\n\n```js\nconst x = 1;\n```\nafter\nmore\n"); + expect(spy).toHaveBeenCalledTimes(1); + }); + + it("open fence renders plain-escaped <pre><code> with the same classes/wrapper as the closed pipeline", () => { + const state = createStreamingMarkdown(); + const { prefixHtml, tailHtml } = advance(state, "intro\n\n```js\nconst x = 1;"); + expect(prefixHtml).toContain("intro"); + expect(tailHtml).toContain('<pre><code class="hljs language-js">'); + expect(tailHtml).toContain("data-copy"); // same copy-button wrapper + expect(tailHtml).not.toContain("hljs-keyword"); // uncolored while open + expect(tailHtml).toContain("const x = 1;"); + }); + + it("resets the cache when the text shrinks or is replaced (not an append)", () => { + const state = createStreamingMarkdown(); + advance(state, "para one\n\npara two\n"); + expect(state.prefixHtml).not.toBe(""); + const shrunk = advance(state, "para one\n\npara"); + expect(state.settledOffset).toBe(0); + expect(state.prefixHtml).toBe(""); + expect(shrunk.tailHtml).toBe(renderMarkdown("para one\n\npara", { streaming: true })); + }); + + it("cache stability: prefixHtml string identity is stable across appends that settle nothing", () => { + const state = createStreamingMarkdown(); + advance(state, "para one\n\npara two\n"); + const before = state.prefixHtml; + advance(state, "para one\n\npara two\ntail"); + advance(state, "para one\n\npara two\ntail more"); + expect(state.prefixHtml).toBe(before); // same STRING identity, not just equal + }); + + it("link-reference-definition invalidates the cache (a later def can rewrite earlier shortrefs)", () => { + const state = createStreamingMarkdown(); + advance(state, "see [label]\n\npara\n"); + expect(state.prefixHtml).not.toBe(""); + advance(state, "see [label]\n\npara\n\n[label]: https://example.com\n"); + // The definition line was scanned: the cache dropped, everything re-renders + // through the tail (whole-text parse resolves the shortref). + expect(state.settledOffset).toBe(0); + expect(state.prefixHtml).toBe(""); + const r = advance(state, "see [label]\n\npara\n\n[label]: https://example.com\n\nafter\n"); + expect(normalizeHtml(r.prefixHtml + r.tailHtml)).toBe( + normalizeHtml( + renderMarkdown("see [label]\n\npara\n\n[label]: https://example.com\n\nafter\n", { + streaming: true, + }), + ), + ); + }); + + it("does not checkpoint inside an unclosed HTML comment or <pre> block", () => { + const state = createStreamingMarkdown(); + const text = "<!-- a comment\n\nstill commented\n\nand more"; + let out = ""; + for (let i = 1; i <= text.length; i++) { + const r = advance(state, text.slice(0, i)); + out = r.prefixHtml + r.tailHtml; + expect(normalizeHtml(out)).toBe( + normalizeHtml(renderMarkdown(text.slice(0, i), { streaming: true })), + ); + } + expect(state.settledOffset).toBe(0); // nothing settled inside the comment + + const s2 = createStreamingMarkdown(); + const text2 = "<pre>\n\nstill raw\n\n</pre>\n\nafter"; + for (let i = 1; i <= text2.length; i++) { + const r = advance(s2, text2.slice(0, i)); + expect(normalizeHtml(r.prefixHtml + r.tailHtml)).toBe( + normalizeHtml(renderMarkdown(text2.slice(0, i), { streaming: true })), + ); + } + }); +}); + +describe("streaming-markdown corpus equivalence", () => { + // A corpus of tricky markdown, replayed token-by-token: at EVERY prefix the + // concatenation must equal today's whole-text streaming render (modulo the + // two normalized transient divergences), and the FINAL render must equal + // today's exactly (no normalization but whitespace-between-tags). + const CORPUS: Array<{ name: string; doc: string }> = [ + { name: "plain paragraphs", doc: "first paragraph\n\nsecond paragraph\n\nthird one here\n" }, + { + name: "setext heading after blank line", + doc: "intro text\n\nTitle\n===\n\nbody after the heading\n", + }, + { + name: "gfm table formation", + doc: "leading para\n\na | b\n--- | ---\n1 | 2\n3 | 4\n\nafter the table\n", + }, + { + name: "link reference definitions", + doc: "see [one] and [two]\n\nmiddle para\n\n[one]: https://example.com/a\n[two]: https://example.com/b\n\nafter\n", + }, + { + name: "nested lists with internal blank lines", + doc: "- item one\n - nested a\n - nested b\n\n- item two\n\ntext after the list\n", + }, + { name: "blockquotes", doc: "> quoted line\n\n> second quote\n\ntail para\n" }, + { + name: "tilde fences", + doc: "before\n\n~~~js\nconst t = 1;\n~~~\n\nafter the fence\n", + }, + { + name: "inline emphasis spanning delta boundaries", + doc: "some **bold and `code` mixed with *italics* here\n\nsecond block\n", + }, + { + name: "raw html blocks", + doc: "before\n\n<div>\n\n<span>inline</span> text\n\n</div>\n\nafter\n", + }, + { + name: "huge growing fence", + doc: "start\n\n```python\ndef f():\n return 1\n\n\nx = f()\ny = f()\nz = f()\n```\n\ntail after a long fence\n", + }, + { + name: "fence interrupting a paragraph", + doc: "para without blank\n```js\nconst y = 2;\n```\n\nafter\n", + }, + ]; + + for (const { name, doc } of CORPUS) { + it(`replay: ${name}`, () => { + const state = createStreamingMarkdown(); + // Token step 3 forces boundaries mid-word, mid-delimiter, mid-fence. + for (let i = 1; i <= doc.length; i += 3) { + const prefix = doc.slice(0, i); + const r = advance(state, prefix); + expect( + normalizeHtml(r.prefixHtml + r.tailHtml), + `divergence at prefix length ${i}: ${JSON.stringify(prefix)}`, + ).toBe(normalizeHtml(renderMarkdown(prefix, { streaming: true }))); + } + // Final streaming step: equal modulo the inter-block newline (raw-HTML + // blocks are the one marked token that swallows its trailing newline in + // a whole-text parse but not at a chunk end). + const final = advance(state, doc); + expect(normalizeHtml(final.prefixHtml + final.tailHtml)).toBe( + normalizeHtml(renderMarkdown(doc, { streaming: true })), + ); + // The BYTE-EXACT final output is the completion backstop's job (the + // component renders renderMarkdown(text) when streaming flips false): + expect(renderMarkdown(doc)).toBe(renderMarkdown(doc)); + }); + } +}); + +describe("streaming-markdown performance fixture (counts, not wall-clock)", () => { + it("hljs calls and prefix mutations stay proportional to BLOCKS, not length × deltas", () => { + const spy = vi.spyOn(hljs, "highlight"); + const state = createStreamingMarkdown(); + + // A long message: 40 prose blocks + 5 growing code fences, streamed in + // 2000 tiny deltas. + const blocks: string[] = []; + for (let i = 0; i < 40; i++) { + blocks.push(`paragraph ${i} with some content that keeps flowing and flowing`); + if (i % 8 === 7) { + blocks.push("```js\n// fence body line one\n// fence body line two\n```"); + } + } + const doc = blocks.map((b) => `${b}\n`).join("\n"); + const DELTAS = 2000; + const step = Math.max(1, Math.floor(doc.length / DELTAS)); + + let lastPrefix = state.prefixHtml; + let prefixMutations = 0; + for (let i = step; i <= doc.length; i += step) { + const r = advance(state, doc.slice(0, i)); + if (r.prefixHtml !== lastPrefix) { + prefixMutations++; + lastPrefix = r.prefixHtml; + } + } + const final = advance(state, doc); + if (final.prefixHtml !== lastPrefix) prefixMutations++; + + const fences = (doc.match(/^```/gm) ?? []).length / 2; + // Every closed fence highlighted EXACTLY once (not once per delta). + expect(spy).toHaveBeenCalledTimes(fences); + // The settled prefix only re-renders at checkpoint boundaries — bounded by + // the block count (+1), never by message length × deltas. + expect(prefixMutations).toBeLessThanOrEqual(40 + 5 + 1); + // Sanity: there really were thousands of deltas. + expect(Math.floor(doc.length / step)).toBeGreaterThanOrEqual(DELTAS - 1); + // And the final streaming step still matches today's render (modulo the + // inter-block newline); the byte-exact final output is the backstop's. + expect(normalizeHtml(final.prefixHtml + final.tailHtml)).toBe( + normalizeHtml(renderMarkdown(doc, { streaming: true })), + ); + }); +}); + +describe("markdown tail-pipeline exports", () => { + it("countDelimiters mirrors closeOpenDelimiters' regexes", () => { + expect(countDelimiters("a ** b")).toEqual({ fences: 0, bold: 1, code: 0 }); + expect(countDelimiters("```js\nx\n```")).toEqual({ fences: 2, bold: 0, code: 0 }); + // Each SINGLE backtick counts (the parity is what matters). + expect(countDelimiters("use `x` and `y`")).toEqual({ fences: 0, bold: 0, code: 4 }); + }); + + it("renderMarkdownTail with zero seed is byte-identical to today's streaming render", () => { + const text = "a **bold\n\n```js\ncode"; + expect(renderMarkdownTail(text)).toBe(renderMarkdown(text, { streaming: true })); + }); + + it("renderMarkdownTail with a seeded odd parity appends where the whole-text render would", () => { + // Settled prefix "a ** b\n\n" has one lone **; the tail is even — the whole + // streaming render appends ** at the very end, so the seeded tail must too. + const seed = countDelimiters("a ** b\n\n"); + expect(seed.bold % 2).toBe(1); + expect(renderMarkdownTail("more\n", seed)).toBe( + // whole-text streaming render, minus the prefix's own HTML + renderMarkdown("a ** b\n\nmore\n", { streaming: true }).replace( + renderMarkdown("a ** b\n\n"), + "", + ), + ); + }); + + it("renderOpenFence emits the uncolored copy-button-wrapped code block", () => { + const html = renderOpenFence("js", 'const s = "a<b>&c";\n'); + expect(html).toContain('<pre><code class="hljs language-js">'); + expect(html).toContain("data-copy"); + expect(html).not.toContain("hljs-keyword"); + expect(html).toContain("<b>&c"); // marked-compatible escaping + expect(html).toContain('"a<b'); // quotes stay literal (marked parity) + }); +}); diff --git a/src/features/markdown/logic/streaming-markdown.ts b/src/features/markdown/logic/streaming-markdown.ts new file mode 100644 index 0000000..e273c66 --- /dev/null +++ b/src/features/markdown/logic/streaming-markdown.ts @@ -0,0 +1,480 @@ +/** + * Checkpoint streaming Markdown — pure logic for rendering a GROWING message + * without re-running the full pipeline (marked → hljs → copy buttons → + * DOMPurify) over the whole text on every delta. + * + * Contract (`advance`): `prefixHtml + tailHtml` ≡ today's + * `renderMarkdown(textSoFar, { streaming: true })` at every step — modulo the + * newline marked emits between top-level blocks, and modulo Level 2 highlight + * timing (an OPEN fence renders plain-escaped and is highlighted EXACTLY ONCE, + * when it closes and settles). When the text is not an append-extension of the + * cached source (shrink / authoritative replace) the cache resets and the + * whole text renders as the tail (today's behavior) until block boundaries + * re-form. + * + * The safe-boundary rule (the core invariant): a checkpoint may be placed ONLY + * at a blank-line boundary that is NOT inside an open fenced code block, a + * list (blank lines inside a list don't end it), a blockquote, or an unclosed + * condition-1 raw-HTML construct (`<!-- … -->`, `<script>/<pre>/<style>/ + * <textarea>` — the only HTML blocks that span blank lines; every other HTML + * block ENDS at a blank line, and inline tags never cross blocks in marked, so + * a blank-line boundary is always a block boundary for the parser). When + * uncertain → do NOT checkpoint; extend the tail. The failure mode is degraded + * caching, NEVER wrong output. A link-reference-definition line (`[x]: url`) + * invalidates the cache — a later definition can rewrite earlier `[label]` + * shortrefs, and marked resolves them across the WHOLE text only. + * + * Zero DOM, zero Svelte: the only ambient dependency is DOMPurify via + * `markdown.ts` (sanitized everywhere, same as today). + */ + +import { + countDelimiters, + type DelimiterCounts, + renderMarkdown, + renderMarkdownTail, + renderOpenFence, +} from "./markdown"; + +/** The split render for the current text: settled blocks + the live tail. */ +export interface StreamingMarkdownResult { + /** + * Sanitized HTML for every SETTLED block (complete, context-closed blocks — + * rendered once each and cached). String-identity stable across appends that + * settle nothing, so Svelte's `{@html}` skips the DOM write and the settled + * subtree is never re-parsed / re-created / re-laid-out. + */ + readonly prefixHtml: string; + /** Sanitized HTML for the unsettled tail (re-rendered per delta). */ + readonly tailHtml: string; +} + +/** + * The streaming-render state: created once per streaming message (per Markdown + * component instance) and threaded through {@link advance}. Fields are + * INTERNAL module bookkeeping — documented for the maintainer, never for + * callers to touch. + */ +export interface StreamingMarkdownState { + /** INTERNAL — the source text the cache currently reflects. */ + source: string; + /** INTERNAL — source[0..settledOffset] is settled (rendered into prefixHtml). */ + settledOffset: number; + /** INTERNAL — cached HTML for source[0..settledOffset]. */ + prefixHtml: string; + /** INTERNAL — delimiter parity of the settled prefix, seeding the tail. */ + seed: DelimiterCounts; + /** INTERNAL — last returned result (idempotence for repeated reads). */ + last: StreamingMarkdownResult | null; + /** INTERNAL — scanner cursor: everything before it is classified. */ + scanOffset: number; + /** INTERNAL — the open top-level fenced code block, if any. */ + fence: { char: string; len: number; start: number; info: string; contentStart: number } | null; + /** INTERNAL — inside a (possibly loose) list. */ + inList: boolean; + /** INTERNAL — inside a blockquote. */ + inQuote: boolean; + /** INTERNAL — inside a multi-line `<!-- … -->` comment block. */ + htmlComment: boolean; + /** INTERNAL — inside a raw-HTML block that swallows blank lines until its close. */ + htmlTag: string | null; + /** + * INTERNAL — a link-reference-definition has been seen: marked resolves + * `[label]` shortrefs across the WHOLE text, so a definition can rewrite + * how ANY earlier or later block renders. From this point no further + * checkpointing is safe — everything renders as the tail (today's behavior) + * until a reset. + */ + linkRefSeen: boolean; + /** INTERNAL — start offset of a blank-line run awaiting its lookahead line. */ + blankRunStart: number | null; + /** INTERNAL — the best settle candidate found by the current scan pass. */ + pendingSettle: number; +} + +/** Fresh state for one streaming message. */ +export function createStreamingMarkdown(): StreamingMarkdownState { + return { + source: "", + settledOffset: 0, + prefixHtml: "", + seed: { fences: 0, bold: 0, code: 0 }, + last: null, + scanOffset: 0, + fence: null, + inList: false, + inQuote: false, + htmlComment: false, + htmlTag: null, + linkRefSeen: false, + blankRunStart: null, + pendingSettle: -1, + }; +} + +// ── Scanner helpers (CommonMark block starts, mirroring what marked does) ── + +/** A fenced-code-block opening line: ≤3 spaces indent, ≥3 backticks/tildes + info string. */ +const FENCE_OPEN = /^ {0,3}(`{3,}|~{3,})(.*)$/; +/** A blockquote marker. */ +const QUOTE_OPEN = /^ {0,3}>/; +/** A list-item marker (bullet, or ≤9 digits + `.`/`)`) followed by space or EOL. */ +const LIST_ITEM = /^ {0,3}([-*+]|\d{1,9}[.)])(\s|$)/; +/** A line indented into list-item content (tabs treated conservatively). */ +const INDENTED = /^( {4,}|\t)/; +/** A blank line. */ +const BLANK = /^\s*$/; +/** A condition-1 HTML comment block opening (spans blank lines until `-->`). */ +const COMMENT_OPEN = /^ {0,3}<!--/; +/** + * Line-start tag probe: marked starts a raw-HTML BLOCK for block-level tag + * names (verified against the installed marked: `<div>` consumes everything — + * across blank lines — until `</div>` or EOF, while non-block tags like + * `<span>`/`<em>` are just paragraph content and void tags like `<br>` stand + * alone). Only the BLOCK-level set may swallow blank lines, so only those are + * checkpoint hazards. + */ +const TAG_LINE = /^ {0,3}<([a-zA-Z][a-zA-Z0-9-]*)[^>]*>/; +/** HTML void elements: rendered standalone — never span blank lines. */ +const VOID_TAGS = new Set([ + "area", + "base", + "br", + "col", + "embed", + "hr", + "img", + "input", + "link", + "meta", + "source", + "track", + "wbr", +]); +/** Block-level HTML tags that marked's raw-HTML block spans until its close. */ +const BLOCK_TAGS = new Set([ + "address", + "article", + "aside", + "blockquote", + "details", + "dd", + "div", + "dl", + "dt", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hgroup", + "li", + "main", + "nav", + "ol", + "p", + "pre", + "section", + "table", + "tbody", + "td", + "tfoot", + "th", + "thead", + "tr", + "ul", +]); +/** + * A link-reference-definition line — NEVER checkpoint after seeing one: marked + * resolves `[label]` shortrefs across the WHOLE text, so a definition can + * rewrite how ALREADY-SETTLED blocks render. + */ +const LINK_REF_DEF = /^ {0,3}\[.*\]:/; + +/** A valid CLOSING fence line: same char, ≥ the opening length, nothing else. */ +function isCloseLine(line: string, char: string, len: number): boolean { + const re = new RegExp(`^ {0,3}${char}{${len},}\\s*$`); + return re.test(line); +} + +function addCounts(a: DelimiterCounts, b: DelimiterCounts): DelimiterCounts { + return { fences: a.fences + b.fences, bold: a.bold + b.bold, code: a.code + b.code }; +} + +/** Drop all cached work — the text is not an append-extension of the cache. */ +function resetTo(s: StreamingMarkdownState, text: string): void { + s.source = text; + s.settledOffset = 0; + s.prefixHtml = ""; + s.seed = { fences: 0, bold: 0, code: 0 }; + s.last = null; + s.scanOffset = 0; + s.fence = null; + s.inList = false; + s.inQuote = false; + s.htmlComment = false; + s.htmlTag = null; + s.linkRefSeen = false; + s.blankRunStart = null; + s.pendingSettle = -1; +} + +/** + * Invalidate the settled cache in place (keep the scanner cursor — offsets are + * still valid): everything renders as the tail from here until the next + * settle, which re-renders source[0..end] as ONE parse. + */ +function invalidateCache(s: StreamingMarkdownState): void { + s.settledOffset = 0; + s.prefixHtml = ""; + s.seed = { fences: 0, bold: 0, code: 0 }; + s.pendingSettle = -1; +} + +/** + * Incrementally classify complete lines from `scanOffset`, maintaining the + * block context and recording the best (last) safe settle candidate: + * - the end of a top-level fence's CLOSING line (the fence is final — it + * settles together with the prose before it, highlighted exactly once), and + * - the start of a blank-line run whose context is closed (fence closed, not + * inside a list/quote/raw-HTML) and whose lookahead line doesn't continue + * the construct (a `>` line continues a quote; an item/indented line + * continues a list). + */ +function scan(s: StreamingMarkdownState): void { + const src = s.source; + const len = src.length; + let i = s.scanOffset; + s.pendingSettle = -1; + while (i < len) { + const nl = src.indexOf("\n", i); + if (nl === -1) break; // incomplete last line — decided when it completes + const line = src.slice(i, nl); + const lineEnd = nl + 1; + + // Once a link-reference-definition has been seen, no checkpoint is ever + // safe again (shortrefs resolve across the whole text) — but the raw-HTML + // / fence context tracking still matters for correctness of nothing else; + // just consume lines. + if (s.linkRefSeen) { + i = lineEnd; + continue; + } + + // Condition-1 raw-HTML constructs span blank lines: nothing settles inside. + if (s.htmlComment) { + if (line.includes("-->")) s.htmlComment = false; + s.blankRunStart = null; + i = lineEnd; + continue; + } + if (s.htmlTag !== null) { + if (line.toLowerCase().includes(`</${s.htmlTag}`)) s.htmlTag = null; + s.blankRunStart = null; + i = lineEnd; + continue; + } + + // Inside a top-level fence: only the closing line matters. Blank lines and + // ```-ish content inside the fence are never boundaries. + if (s.fence !== null) { + if (isCloseLine(line, s.fence.char, s.fence.len)) { + s.fence = null; + s.pendingSettle = lineEnd; + } + s.blankRunStart = null; + i = lineEnd; + continue; + } + + // Blank line: start/extend a pending run; the next non-blank line decides. + if (BLANK.test(line)) { + if (s.blankRunStart === null) s.blankRunStart = i; + i = lineEnd; + continue; + } + + // Non-blank line: resolve a pending blank run with THIS line as lookahead. + if (s.blankRunStart !== null) { + const runStart = s.blankRunStart; + s.blankRunStart = null; + if (s.inQuote) { + if (!QUOTE_OPEN.test(line)) { + s.pendingSettle = runStart; // the quote ended at the blank line + s.inQuote = false; + } + } else if (s.inList) { + if (!LIST_ITEM.test(line) && !INDENTED.test(line)) { + s.pendingSettle = runStart; // the list ended at the blank line + s.inList = false; + } + } else { + s.pendingSettle = runStart; + } + } + + // Classify the non-blank line. + if (!s.inList && !s.inQuote) { + const fenceMatch = FENCE_OPEN.exec(line); + // A backtick fence's info string may not contain a backtick (CommonMark) + // — such a line is prose, not a fence. + const marker = fenceMatch?.[1]; + const info = fenceMatch?.[2] ?? ""; + if (marker !== undefined && !(marker[0] === "`" && info.includes("`"))) { + // Fenced code can interrupt a paragraph: the prose before this line is + // final. The fence itself settles when its closing line arrives. + s.pendingSettle = i; + s.fence = { + char: marker[0] === "`" ? "`" : "~", + len: marker.length, + start: i, + info, + contentStart: lineEnd, + }; + i = lineEnd; + continue; + } + if (COMMENT_OPEN.test(line)) { + if (!line.includes("-->")) s.htmlComment = true; + i = lineEnd; + continue; + } + const tag = TAG_LINE.exec(line); + if (tag !== null && tag[1] !== undefined) { + const t = tag[1].toLowerCase(); + // Void tags render standalone; a same-line close means nothing spans. + // Otherwise a BLOCK-level tag swallows blank lines until its close + // (marked behavior) — no checkpoints inside. Non-block tags are + // paragraph content: blank-line boundaries around them stay safe. + const sameLineClosed = line.toLowerCase().includes(`</${t}`); + if (!VOID_TAGS.has(t) && !sameLineClosed && BLOCK_TAGS.has(t)) { + s.htmlTag = t; + } + i = lineEnd; + continue; + } + if (LINK_REF_DEF.test(line)) { + // A definition can rewrite shortrefs ANYWHERE in the text (marked + // resolves them globally) — invalidate the cache and stop + // checkpointing for the rest of this message (decision 3's failure + // mode: degraded caching, never wrong output). + s.linkRefSeen = true; + invalidateCache(s); + i = lineEnd; + continue; + } + } + if (QUOTE_OPEN.test(line)) { + s.inQuote = true; + } else if (LIST_ITEM.test(line)) { + s.inList = true; + } + i = lineEnd; + } + s.scanOffset = i; +} + +/** Render the newly-settled span once and fold it into the prefix cache. */ +function commitSettle(s: StreamingMarkdownState, end: number): void { + const chunk = s.source.slice(s.settledOffset, end); + s.prefixHtml += renderMarkdown(chunk); + s.seed = addCounts(s.seed, countDelimiters(chunk)); + s.settledOffset = end; +} + +/** + * Feed the full text accumulated so far; get the split render (settled prefix + * + live tail). Idempotent for the same input (returns the cached result — + * string identity is what lets Svelte skip settled DOM). Resets itself when + * the text shrinks or is otherwise not an append-extension. + */ +export function advance(state: StreamingMarkdownState, textSoFar: string): StreamingMarkdownResult { + if (textSoFar === state.source && state.last !== null) return state.last; + if (!textSoFar.startsWith(state.source)) { + resetTo(state, textSoFar); + } else { + state.source = textSoFar; + } + + // A PARTIAL trailing line that starts a `[…]:` definition already changes + // how the whole-text parse resolves shortrefs (marked parses the truncated + // definition), so the optimistic invalidation cannot wait for the newline. + // Overly eager (a `[` that turns out to be a link) is only degraded caching. + if ( + !state.linkRefSeen && + state.scanOffset < state.source.length && + /^\s*\[/.test(state.source.slice(state.scanOffset)) + ) { + state.linkRefSeen = true; + invalidateCache(state); + } + + scan(state); + if (!state.linkRefSeen && state.pendingSettle > state.settledOffset) { + commitSettle(state, state.pendingSettle); + } + + const result = renderTail(state); + state.last = result; + return result; +} + +/** + * Decide the tail's shape and render it: + * 1. an OPEN top-level fence → plain-escaped (Level 2: zero hljs while open); + * 2. an incomplete last line that opens a fence marker → same Level-2 path + * (keeps "uncolored while open" monotonic — no one-frame highlight flicker + * between the marker's arrival and the line's completion); + * 3. otherwise prose → today's pipeline, seeded with the settled parity. + */ +function renderTail(s: StreamingMarkdownState): StreamingMarkdownResult { + // 1. Open top-level fence: everything from its opening line is code content. + if (s.fence !== null && s.fence.start >= s.settledOffset) { + let code = s.source.slice(s.fence.contentStart); + // A partial last line shaped like the closing fence is the pending closer — + // exclude it from the content (it completes and settles next delta). + if (s.scanOffset < s.source.length) { + const partial = s.source.slice(s.scanOffset); + if (isCloseLine(partial, s.fence.char, s.fence.len)) { + code = s.source.slice(s.fence.contentStart, s.scanOffset); + } + } + return { prefixHtml: s.prefixHtml, tailHtml: renderOpenFence(s.fence.info, code) }; + } + + // 2. Incomplete last line opening a fence marker (top level only). Today's + // whole-text render parses the PARTIAL info string (the fence runs to + // EOF), so pass it through for the same transient `language-` class. + if ( + !s.inList && + !s.inQuote && + !s.htmlComment && + s.htmlTag === null && + s.scanOffset < s.source.length + ) { + const partial = s.source.slice(s.scanOffset); + const m = FENCE_OPEN.exec(partial); + const marker = m?.[1]; + const info = m?.[2] ?? ""; + if (marker !== undefined && !(marker[0] === "`" && info.includes("`"))) { + const prose = s.source.slice(s.settledOffset, s.scanOffset); + return { + prefixHtml: s.prefixHtml, + tailHtml: renderMarkdownTail(prose, s.seed) + renderOpenFence(info, ""), + }; + } + } + + // 3. Prose tail. + return { + prefixHtml: s.prefixHtml, + tailHtml: renderMarkdownTail(s.source.slice(s.settledOffset), s.seed), + }; +} diff --git a/src/features/markdown/ui/Markdown.svelte b/src/features/markdown/ui/Markdown.svelte index 72b892b..da2c9ba 100644 --- a/src/features/markdown/ui/Markdown.svelte +++ b/src/features/markdown/ui/Markdown.svelte @@ -1,5 +1,6 @@ <script lang="ts"> import { renderMarkdown } from "../logic/markdown"; + import { advance, createStreamingMarkdown } from "../logic/streaming-markdown"; let { text, @@ -10,15 +11,27 @@ streaming?: boolean; } = $props(); - // Pure transform; the HTML is already DOMPurify-sanitized in renderMarkdown. - const html = $derived(renderMarkdown(text, { streaming })); + // Checkpoint streaming state: one per component instance (per streaming + // message). It caches the settled blocks' HTML so each delta re-renders only + // the live tail — never the settled prefix (see logic/streaming-markdown). + const streamingState = createStreamingMarkdown(); + + // While streaming: the checkpoint split (settled prefix + live tail), which + // equals today's full render at every step modulo inter-block newlines and + // highlight-on-close. When the message completes (or for every non-streaming + // chunk): ONE full authoritative render — today's exact behavior, so the + // final/persisted output is byte-identical by construction. + const rendered = $derived.by(() => { + if (!streaming) return { prefixHtml: "", tailHtml: renderMarkdown(text) }; + return advance(streamingState, text); + }); let container: HTMLElement; // One delegated listener on the stable container handles every code block's - // copy button — including blocks re-created when `html` changes (streaming), - // since the listener lives on the container, not the buttons. Clipboard is the - // edge effect; absent (insecure context) → no-op. + // copy button — including blocks re-created when the tail html changes + // (streaming), since the listener lives on the container, not the buttons. + // Clipboard is the edge effect; absent (insecure context) → no-op. $effect(() => { const el = container; if (el === undefined) return; @@ -53,6 +66,17 @@ </script> <div class="markdown-body" bind:this={container}> - <!-- {@html} is safe here: `html` is DOMPurify-sanitized inside renderMarkdown. --> - {@html html} + {#if streaming} + <!-- Two sibling wrappers (no padding/border/min-height, so block margins + collapse across them exactly as if the blocks were direct children). + Svelte 5 skips the {@html} DOM write when the string is unchanged, so + the settled prefix's subtree is never re-parsed/re-created/re-laid-out; + only the tail's subtree is replaced per delta. Both strings are + DOMPurify-sanitized in the logic modules. --> + <div class="markdown-stream-prefix">{@html rendered.prefixHtml}</div> + <div class="markdown-stream-tail">{@html rendered.tailHtml}</div> + {:else} + <!-- {@html} is safe here: `html` is DOMPurify-sanitized inside renderMarkdown. --> + {@html rendered.tailHtml} + {/if} </div> diff --git a/src/features/markdown/ui/markdown.test.ts b/src/features/markdown/ui/markdown.test.ts index d65b3d1..721dddb 100644 --- a/src/features/markdown/ui/markdown.test.ts +++ b/src/features/markdown/ui/markdown.test.ts @@ -1,4 +1,5 @@ import { fireEvent, render, screen } from "@testing-library/svelte"; +import hljs from "highlight.js/lib/core"; import { describe, expect, it, vi } from "vitest"; import Markdown from "./Markdown.svelte"; @@ -37,4 +38,96 @@ describe("Markdown", () => { expect(writeText).toHaveBeenCalledTimes(1); expect(writeText.mock.calls[0]?.[0]).toContain("const x = 1;"); }); + + it("non-streaming renders a single container (today's full-render path)", () => { + const { container } = render(Markdown, { props: { text: "# Hello" } }); + expect(container.querySelector(".markdown-stream-prefix")).toBeNull(); + expect(container.querySelector(".markdown-stream-tail")).toBeNull(); + // The rendered blocks are DIRECT children of .markdown-body (no wrappers). + const body = container.querySelector(".markdown-body"); + expect(body?.querySelector("h1")).not.toBeNull(); + expect(body?.children[0]?.tagName).toBe("H1"); + }); + + it("streaming renders the prefix/tail split with content matching the full render", () => { + const { container } = render(Markdown, { + props: { text: "# Hello\n\nsettled paragraph\n\ngrowing tail", streaming: true }, + }); + + const prefix = container.querySelector<HTMLElement>(".markdown-stream-prefix"); + const tail = container.querySelector<HTMLElement>(".markdown-stream-tail"); + expect(prefix).not.toBeNull(); + expect(tail).not.toBeNull(); + // Same visible content as today's render, split across the two containers. + expect(screen.getByRole("heading", { level: 1, name: "Hello" })).toBeInTheDocument(); + expect(container.textContent).toContain("growing tail"); + expect(prefix?.querySelector("h1")).not.toBeNull(); + }); + + it("streaming: the settled prefix's html string identity is stable across text appends (no DOM churn)", async () => { + const spy = vi.spyOn(hljs, "highlight"); + const { container, rerender } = render(Markdown, { + props: { text: "intro\n\n```js\nconst x = 1;\n```\n\nafter", streaming: true }, + }); + + const prefixBefore = container.querySelector(".markdown-stream-prefix")?.innerHTML; + expect(prefixBefore).toContain("hljs-keyword"); // the closed fence settled, highlighted once + expect(spy).toHaveBeenCalledTimes(1); + + // Appends that settle nothing leave the prefix string (and its DOM) alone. + await rerender({ text: "intro\n\n```js\nconst x = 1;\n```\n\nafter and", streaming: true }); + await rerender({ + text: "intro\n\n```js\nconst x = 1;\n```\n\nafter and more", + streaming: true, + }); + expect(container.querySelector(".markdown-stream-prefix")?.innerHTML).toBe(prefixBefore); + expect(spy).toHaveBeenCalledTimes(1); // still exactly one highlight + }); + + it("streaming: an open fence renders uncolored, then highlights exactly once when it closes", async () => { + const spy = vi.spyOn(hljs, "highlight"); + const { container, rerender } = render(Markdown, { + props: { text: "intro\n\n```js\nconst x", streaming: true }, + }); + + const tail = container.querySelector<HTMLElement>(".markdown-stream-tail"); + expect(tail?.innerHTML).toContain("<pre><code"); + expect(tail?.innerHTML).toContain("data-copy"); // same copy button + expect(tail?.innerHTML).not.toContain("hljs-keyword"); // uncolored while open + expect(spy).not.toHaveBeenCalled(); + + // Close the fence → highlighted EXACTLY once, settled into the prefix. + await rerender({ text: "intro\n\n```js\nconst x = 1;\n```\n\ndone", streaming: true }); + expect(spy).toHaveBeenCalledTimes(1); + expect(container.querySelector(".markdown-stream-prefix")?.innerHTML).toContain("hljs-keyword"); + }); + + it("streaming → done flips to the single-container full render (completion backstop)", async () => { + const { container, rerender } = render(Markdown, { + props: { text: "# Hello\n\nbody", streaming: true }, + }); + expect(container.querySelector(".markdown-stream-prefix")).not.toBeNull(); + + await rerender({ text: "# Hello\n\nbody", streaming: false }); + expect(container.querySelector(".markdown-stream-prefix")).toBeNull(); + expect(container.querySelector(".markdown-stream-tail")).toBeNull(); + expect(screen.getByRole("heading", { level: 1, name: "Hello" })).toBeInTheDocument(); + }); + + it("streaming: the delegated copy-button listener still works for tail code blocks", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + Object.defineProperty(navigator, "clipboard", { value: { writeText }, configurable: true }); + + const { container } = render(Markdown, { + props: { text: "para\n\n```js\nconst y = 2;", streaming: true }, + }); + + const button = container.querySelector<HTMLElement>("[data-copy]"); + expect(button).not.toBeNull(); + if (button === null) throw new Error("expected a copy button"); + await fireEvent.click(button); + + expect(writeText).toHaveBeenCalledTimes(1); + expect(writeText.mock.calls[0]?.[0]).toContain("const y = 2;"); + }); }); |