summaryrefslogtreecommitdiffhomepage
path: root/src/core/chunks
diff options
context:
space:
mode:
Diffstat (limited to 'src/core/chunks')
-rw-r--r--src/core/chunks/image-url.test.ts46
-rw-r--r--src/core/chunks/image-url.ts35
-rw-r--r--src/core/chunks/index.ts1
-rw-r--r--src/core/chunks/reducer.test.ts153
-rw-r--r--src/core/chunks/reducer.ts153
5 files changed, 360 insertions, 28 deletions
diff --git a/src/core/chunks/image-url.test.ts b/src/core/chunks/image-url.test.ts
new file mode 100644
index 0000000..8a79c09
--- /dev/null
+++ b/src/core/chunks/image-url.test.ts
@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import { resolveImageUrl } from "./image-url";
+
+const BASE = "http://localhost:24203";
+
+describe("resolveImageUrl", () => {
+ it("returns a data URL as-is (the optimistic echo / a pasted image)", () => {
+ const dataUrl = "data:image/png;base64,iVBORw0KGgo=";
+ expect(resolveImageUrl(dataUrl, BASE)).toBe(dataUrl);
+ });
+
+ it("returns an absolute http URL as-is", () => {
+ const abs = "https://example.com/img.png";
+ expect(resolveImageUrl(abs, BASE)).toBe(abs);
+ });
+
+ it("prepends the api base to a relative /images/ path", () => {
+ expect(resolveImageUrl("/images/conv-123/abc-456.png", BASE)).toBe(
+ "http://localhost:24203/images/conv-123/abc-456.png",
+ );
+ });
+
+ it("does not double the slash when the base has a trailing slash", () => {
+ expect(resolveImageUrl("/images/c/x.png", "http://localhost:24203/")).toBe(
+ "http://localhost:24203/images/c/x.png",
+ );
+ });
+
+ it("adds a leading slash to a path-relative url without one", () => {
+ expect(resolveImageUrl("images/c/x.png", BASE)).toBe("http://localhost:24203/images/c/x.png");
+ });
+
+ it("returns the relative path as-is when apiBase is empty (root-relative)", () => {
+ // A browser resolves a root-relative `/images/…` against the document origin.
+ expect(resolveImageUrl("/images/c/x.png", "")).toBe("/images/c/x.png");
+ });
+
+ it("handles a relative path with an empty apiBase (path-relative without slash)", () => {
+ expect(resolveImageUrl("images/c/x.png", "")).toBe("/images/c/x.png");
+ });
+
+ it("returns a data URL as-is even with an empty apiBase", () => {
+ const dataUrl = "data:image/jpeg;base64,AAAA";
+ expect(resolveImageUrl(dataUrl, "")).toBe(dataUrl);
+ });
+});
diff --git a/src/core/chunks/image-url.ts b/src/core/chunks/image-url.ts
new file mode 100644
index 0000000..e5ec756
--- /dev/null
+++ b/src/core/chunks/image-url.ts
@@ -0,0 +1,35 @@
+/**
+ * Resolve an `ImageChunk.url` into a renderable `<img src>` value.
+ *
+ * Persisted image chunks now carry a COMPACT HTTP path
+ * (`/images/<conversationId>/<uuid>.png`) served by the backend — NOT a base64
+ * data URL (images are stored on disk under tmp, not in the conversation store,
+ * to keep SQLite payloads small). The optimistic echo (what the FE just sent in
+ * `ChatRequest.images`) still carries a data URL, and a chunk could also carry
+ * an absolute `http(s)://` URL, so the resolution is format-aware:
+ *
+ * - `data:` URL → returned as-is (the optimistic echo / a pasted data URL).
+ * - `http(s)://` → returned as-is (an absolute URL already).
+ * - anything else (a relative path like `/images/…`) → `apiBase` is prepended
+ * (with no double slash). An empty `apiBase` leaves a root-relative path,
+ * which a browser resolves against the document origin.
+ *
+ * Pure: input → output, zero DOM, zero Svelte.
+ *
+ * @param url The chunk's `url` (data URL, absolute, or relative path).
+ * @param apiBase The HTTP API base URL (e.g. `http://localhost:24203`).
+ */
+export function resolveImageUrl(url: string, apiBase: string): string {
+ if (url.startsWith("data:") || url.startsWith("http://") || url.startsWith("https://")) {
+ return url;
+ }
+ // A relative path (e.g. `/images/…`) — normalize to a leading slash and
+ // prepend the api base. With an empty base this yields a root-relative path
+ // (a browser resolves `/images/…` against the document origin).
+ const path = url.startsWith("/") ? url : `/${url}`;
+ if (apiBase.length === 0) return path;
+ // Join without a double slash: strip a trailing slash from the base, then
+ // append the (leading-slash) path verbatim.
+ const base = apiBase.endsWith("/") ? apiBase.slice(0, -1) : apiBase;
+ return `${base}${path}`;
+}
diff --git a/src/core/chunks/index.ts b/src/core/chunks/index.ts
index eea2303..bdd6ce4 100644
--- a/src/core/chunks/index.ts
+++ b/src/core/chunks/index.ts
@@ -1,5 +1,6 @@
export type { RenderGroup, ToolBatchEntry } from "./groups";
export { groupRenderedChunks } from "./groups";
+export { resolveImageUrl } from "./image-url";
export {
appendUserMessage,
applyHistory,
diff --git a/src/core/chunks/reducer.test.ts b/src/core/chunks/reducer.test.ts
index 8a2e1b7..058552e 100644
--- a/src/core/chunks/reducer.test.ts
+++ b/src/core/chunks/reducer.test.ts
@@ -1,4 +1,5 @@
import type {
+ ImageInput,
StepId,
StoredChunk,
TurnDoneEvent,
@@ -869,3 +870,155 @@ describe("appendUserMessage", () => {
expect(s.provisional[1]?.chunk).toEqual({ type: "text", text: "user msg" });
});
});
+
+const PNG = "data:image/png;base64,AAAA";
+const JPG = "data:image/jpeg;base64,BBBB";
+
+describe("appendUserMessage — images (vision handoff)", () => {
+ it("echoes text then image chunks in order", () => {
+ const images: ImageInput[] = [
+ { url: PNG, mimeType: "image/png" },
+ { url: JPG, mimeType: "image/jpeg" },
+ ];
+ let s = initialState();
+ s = appendUserMessage(s, "what's this?", images);
+ const chunks = selectChunks(s);
+ expect(chunks).toHaveLength(3);
+ expect(chunks[0]?.role).toBe("user");
+ expect(chunks[0]?.chunk).toEqual({ type: "text", text: "what's this?" });
+ expect(chunks[1]?.chunk).toEqual({ type: "image", url: PNG, mimeType: "image/png" });
+ expect(chunks[2]?.chunk).toEqual({ type: "image", url: JPG, mimeType: "image/jpeg" });
+ expect(chunks.every((c) => c.provisional && c.seq === null)).toBe(true);
+ });
+
+ it("omits mimeType when not provided", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "look", [{ url: PNG }]);
+ const img = selectChunks(s)[1]?.chunk;
+ expect(img).toEqual({ type: "image", url: PNG });
+ expect(img).not.toHaveProperty("mimeType");
+ });
+
+ it("echoes images-only when text is empty (no text chunk)", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "", [{ url: PNG, mimeType: "image/png" }]);
+ const chunks = selectChunks(s);
+ expect(chunks).toHaveLength(1);
+ expect(chunks[0]?.chunk.type).toBe("image");
+ });
+
+ it("echoes nothing for empty text + empty images", () => {
+ let s = initialState();
+ s = foldEvent(s, turnStart("t1"));
+ s = foldEvent(s, textDelta("t1", "partial"));
+ s = appendUserMessage(s, "", []);
+ // Defensive flush still happened; no user chunk added.
+ const users = selectChunks(s).filter((c) => c.role === "user");
+ expect(users).toHaveLength(0);
+ expect(s.accumulating).toBeNull();
+ });
+
+ it("skips images with an empty url", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "hi", [
+ { url: "", mimeType: "image/png" },
+ { url: PNG, mimeType: "image/png" },
+ ]);
+ const chunks = selectChunks(s);
+ expect(chunks).toHaveLength(2); // text + the one valid image
+ expect(chunks[1]?.chunk.type).toBe("image");
+ });
+
+ it("groups text + images into one user ChatMessage", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "see this", [{ url: PNG }]);
+ const msgs = selectMessages(s);
+ expect(msgs).toHaveLength(1);
+ expect(msgs[0]?.role).toBe("user");
+ expect(msgs[0]?.chunks).toHaveLength(2);
+ });
+});
+
+describe("foldEvent — user-message dedups against a text+image echo", () => {
+ const userMessage = (text: string): TurnInputEvent => ({
+ type: "user-message",
+ conversationId: "c1",
+ turnId: "t1",
+ text,
+ });
+
+ it("does not duplicate the text when images follow it in the echo", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "hi", [{ url: PNG, mimeType: "image/png" }]);
+ expect(selectChunks(s)).toHaveLength(2); // text + image
+ s = foldEvent(s, userMessage("hi"));
+ const users = selectChunks(s).filter((c) => c.role === "user");
+ expect(users).toHaveLength(2); // unchanged — no duplicate text
+ expect(users[0]?.chunk.type).toBe("text");
+ expect(users[1]?.chunk.type).toBe("image");
+ expect(s.generating).toBe(true);
+ });
+
+ it("still appends when the echoed text differs", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "first", [{ url: PNG }]);
+ s = foldEvent(s, userMessage("second"));
+ const users = selectChunks(s).filter((c) => c.role === "user");
+ expect(users.filter((c) => c.chunk.type === "text")).toHaveLength(2);
+ });
+});
+
+describe("applyHistory — multi-chunk image echo is superseded by committed", () => {
+ it("drops the provisional [text, image] echo when committed arrives during generation", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "hi", [{ url: PNG, mimeType: "image/png" }]);
+ s = foldEvent(s, turnStart("t1"));
+ expect(s.provisional).toHaveLength(2);
+
+ s = applyHistory(s, [
+ storedChunk(1, "user", { type: "text", text: "hi" }),
+ storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }),
+ ]);
+ expect(s.provisional).toEqual([]);
+ expect(s.committed).toHaveLength(2);
+ expect(s.committed[0]?.chunk).toEqual({ type: "text", text: "hi" });
+ expect(s.committed[1]?.chunk).toEqual({ type: "image", url: PNG, mimeType: "image/png" });
+ });
+
+ it("keeps the echo when committed is only a partial match (img not yet persisted)", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "hi", [
+ { url: PNG, mimeType: "image/png" },
+ { url: JPG, mimeType: "image/jpeg" },
+ ]);
+ s = foldEvent(s, turnStart("t1"));
+ // Only the text + first image have been persisted so far.
+ s = applyHistory(s, [
+ storedChunk(1, "user", { type: "text", text: "hi" }),
+ storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }),
+ ]);
+ // Echo is [text, img1, img2]; committed run is [text, img1] — not a full
+ // match (echo is longer) → keep the echo (turn-seal will drop it wholesale).
+ expect(s.provisional.length).toBeGreaterThan(0);
+ });
+
+ it("renders committed image chunks from history (a non-vision transcription turn)", () => {
+ // The server persists the original image chunk AND the transcription text
+ // in the SAME user message: render both (image, then analysis text).
+ let s = initialState();
+ s = applyHistory(s, [
+ storedChunk(1, "user", { type: "text", text: "describe this" }),
+ storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }),
+ storedChunk(3, "user", {
+ type: "text",
+ text: "[Image analysis (via kimi/k2)]: a red square",
+ }),
+ storedChunk(4, "assistant", { type: "text", text: "the square is red" }),
+ ]);
+ const msgs = selectMessages(s);
+ expect(msgs).toHaveLength(2); // one user message (3 chunks) + one assistant
+ expect(msgs[0]?.role).toBe("user");
+ expect(msgs[0]?.chunks).toHaveLength(3);
+ expect(msgs[0]?.chunks[1]).toEqual({ type: "image", url: PNG, mimeType: "image/png" });
+ });
+});
diff --git a/src/core/chunks/reducer.ts b/src/core/chunks/reducer.ts
index 2152de3..64e41b9 100644
--- a/src/core/chunks/reducer.ts
+++ b/src/core/chunks/reducer.ts
@@ -1,4 +1,5 @@
-import type { AgentEvent, Chunk, StoredChunk } from "@dispatch/wire";
+import type { AgentEvent, Chunk, ImageInput, Role, StoredChunk } from "@dispatch/wire";
+import { assertChunkExhaustive } from "../wire/conformance";
import type { AccumulatingChunk, ProvisionalChunk, TranscriptState } from "./types";
/** The initial empty transcript state. */
@@ -43,6 +44,63 @@ function flushAccumulating(
}
/**
+ * Content equality for two chunks of the SAME role (used to de-dup an
+ * optimistic echo against the authoritative committed version). Compares the
+ * discriminating payload only — `stepId` (generation provenance) is ignored
+ * since it is absent on provisional echoes but present on committed tool chunks.
+ */
+function chunkContentEquals(a: Chunk, b: Chunk): boolean {
+ if (a.type !== b.type) return false;
+ switch (a.type) {
+ case "text": {
+ const o = b as Extract<Chunk, { type: "text" }>;
+ return a.text === o.text;
+ }
+ case "thinking": {
+ const o = b as Extract<Chunk, { type: "thinking" }>;
+ return a.text === o.text;
+ }
+ case "image": {
+ const o = b as Extract<Chunk, { type: "image" }>;
+ return a.url === o.url;
+ }
+ case "error": {
+ const o = b as Extract<Chunk, { type: "error" }>;
+ return a.message === o.message && a.code === o.code;
+ }
+ case "system": {
+ const o = b as Extract<Chunk, { type: "system" }>;
+ return a.text === o.text;
+ }
+ case "tool-call": {
+ const o = b as Extract<Chunk, { type: "tool-call" }>;
+ return a.toolCallId === o.toolCallId && a.toolName === o.toolName;
+ }
+ case "tool-result": {
+ const o = b as Extract<Chunk, { type: "tool-result" }>;
+ return a.toolCallId === o.toolCallId && a.toolName === o.toolName && a.isError === o.isError;
+ }
+ default:
+ return assertChunkExhaustive(a) === assertChunkExhaustive(b);
+ }
+}
+
+/**
+ * The trailing run of consecutive same-role provisional chunks at the end of
+ * `provisional` (the optimistic echo of one message). Returns the start/end
+ * indices `[start, end)` into `provisional` (empty if none).
+ */
+function trailingRun(
+ provisional: readonly ProvisionalChunk[],
+ role: Role,
+): readonly ProvisionalChunk[] {
+ const end = provisional.length;
+ let start = end;
+ while (start > 0 && provisional[start - 1]?.role === role) start--;
+ return provisional.slice(start, end);
+}
+
+/**
* Merge authoritative seq-keyed chunks into the committed history.
* Dedupes by seq (new wins), keeps seq-monotonic order, idempotent.
* When sealedTurnId is set, drops all provisional chunks (now superseded)
@@ -78,21 +136,30 @@ export function applyHistory(
// During generation: if new committed chunks arrived, the provisional
// array may contain duplicates — the optimistic echo from `appendUserMessage`
- // is now backed by a committed chunk (CR-6: user message persisted at turn
- // start). Remove provisional chunks that match the last committed chunk
- // (role + chunk content), keeping only the accumulating (streaming) chunk.
+ // is now backed by committed chunks (CR-6: user message persisted at turn
+ // start). A user message may be multi-chunk (`[text, image, image, …]`), so
+ // match the trailing provisional user-run against the trailing committed
+ // user-run by content equality and drop the whole echo when it is fully
+ // backed. Leaves the accumulating (streaming) chunk untouched.
if (addedNew && state.generating && state.provisional.length > 0) {
- const lastCommitted = committed[committed.length - 1];
- if (lastCommitted !== undefined) {
- const provisional = state.provisional.filter((p) => {
- if (p.role !== lastCommitted.role) return true;
- if (p.chunk.type !== lastCommitted.chunk.type) return true;
- if (p.chunk.type === "text" && lastCommitted.chunk.type === "text") {
- return p.chunk.text !== lastCommitted.chunk.text;
- }
- return true;
- });
- return { ...state, committed, provisional, accumulating: state.accumulating };
+ const provRun = trailingRun(state.provisional, "user");
+ if (provRun.length > 0) {
+ const commRun = trailingRun(committed, "user");
+ // Drop the provisional user echo iff every echoed chunk is content-equal
+ // to the corresponding committed chunk (the server persisted the same
+ // message). A partial match (echo longer than committed) keeps the echo
+ // — the not-yet-committed tail stays until the turn seals.
+ const fullyBacked =
+ commRun.length >= provRun.length &&
+ provRun.every((p, i) => {
+ const c = commRun[i];
+ return c !== undefined && chunkContentEquals(p.chunk, c.chunk);
+ });
+ if (fullyBacked) {
+ const dropStart = state.provisional.length - provRun.length;
+ const provisional = state.provisional.slice(0, dropStart);
+ return { ...state, committed, provisional, accumulating: state.accumulating };
+ }
}
}
@@ -138,16 +205,18 @@ function reduceEvent(state: TranscriptState, event: AgentEvent): TranscriptState
// The turn's USER prompt, surfaced on the event stream (backend CR-3) so a
// WATCHER/late-joiner renders it mid-turn instead of waiting for seal. The
// SENDER already echoed its own prompt optimistically (`appendUserMessage`),
- // so DE-DUP: skip if the trailing provisional chunk is already an identical
- // user text chunk. A pure watcher has no such echo → it appends and renders.
+ // so DE-DUP: skip if the trailing provisional USER run already contains an
+ // identical user text chunk. The echo may be multi-chunk (`[text, image,
+ // image, …]` — `appendUserMessage` appends the text first, then images), so
+ // we scan the whole trailing user run, not just the last chunk (which would
+ // be an image when images were pasted). A pure watcher has no such echo →
+ // it appends and renders. The `user-message` event carries ONLY text (never
+ // images — images arrive via history/loadSince); an images-only send (empty
+ // text) emits no `user-message` and is not de-duped here.
if (event.text.length === 0) return state;
- const last = state.provisional[state.provisional.length - 1];
- if (
- last !== undefined &&
- last.role === "user" &&
- last.chunk.type === "text" &&
- last.chunk.text === event.text
- ) {
+ const run = trailingRun(state.provisional, "user");
+ const alreadyEchoed = run.some((p) => p.chunk.type === "text" && p.chunk.text === event.text);
+ if (alreadyEchoed) {
return { ...state, generating: true };
}
const provisional = flushAccumulating(state.provisional, state.accumulating);
@@ -335,15 +404,43 @@ export function foldEvent(state: TranscriptState, event: AgentEvent): Transcript
/**
* Optimistically append a user message to the provisional list.
* Flushes any in-progress accumulating chunk first (defensively).
- * The provisional user chunk is superseded when applyHistory receives
+ * The provisional user chunks are superseded when applyHistory receives
* the authoritative committed chunks after a turn seals.
+ *
+ * When `images` are provided, they are appended AFTER the text chunk (in
+ * order) as `image` chunks — matching the server's persisted layout
+ * (`[text, image, image, …]` in one user message). A text chunk is only
+ * appended when `text` is non-empty (an images-only send echoes just the
+ * images). The `user-message` event carries only text (never images), so its
+ * de-dup scans the trailing user run for the echoed text rather than just the
+ * last chunk.
*/
-export function appendUserMessage(state: TranscriptState, text: string): TranscriptState {
+export function appendUserMessage(
+ state: TranscriptState,
+ text: string,
+ images?: readonly ImageInput[],
+): TranscriptState {
const provisional = flushAccumulating(state.provisional, state.accumulating);
- const userChunk: Chunk = { type: "text", text };
+ const userChunks: Chunk[] = [];
+ if (text.length > 0) userChunks.push({ type: "text", text });
+ if (images !== undefined) {
+ for (const img of images) {
+ if (img.url.length === 0) continue;
+ const chunk: Chunk =
+ img.mimeType !== undefined
+ ? { type: "image", url: img.url, mimeType: img.mimeType }
+ : { type: "image", url: img.url };
+ userChunks.push(chunk);
+ }
+ }
+ if (userChunks.length === 0) {
+ // Nothing to echo (empty text + no images) — leave state unchanged but
+ // still flush any accumulating chunk defensively.
+ return { ...state, provisional, accumulating: null };
+ }
return {
...state,
- provisional: [...provisional, { role: "user", chunk: userChunk }],
+ provisional: [...provisional, ...userChunks.map((chunk) => ({ role: "user", chunk }) as const)],
accumulating: null,
};
}
generated by cgit v1.2.3 (git 2.39.1) at 2026-07-11 16:16:38 +0000