From 3566a20ebbded754070fce66af48690d1a904879 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 04:18:59 +0900 Subject: feat(vision): image paste + transcript image rendering + vision badge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vision & vision-handoff frontend (consumes the backend's additive wire@0.12.0 / transport-contract@0.22.0 image types — no version bump). Contracts mirrored: - .dispatch/wire.reference.md: ImageChunk added to the Chunk union + ImageChunk/ImageInput interfaces. - .dispatch/transport-contract.reference.md: ChatRequest.images, ModelMetadata.vision, + ImageChunk/ImageInput re-exports. Core (core/chunks): - conformance: assertChunkExhaustive handles the new 'image' variant (the guard caught it — its purpose). - appendUserMessage(state, text, images?) echoes a [text, image, ...] user run; the user-message event dedup scans the trailing user run (not just the last chunk) so an image-bearing echo doesn't duplicate the text; applyHistory's during-gen dedup matches a multi-chunk echo by content equality (chunkContentEquals + trailingRun helpers). UI: - ChatView renders user 'image' chunks as lazy bubbles; a non-vision model's persisted [image, analysis-text] both render. read_image tool renders generically (no special-casing). - Composer: clipboard paste / file picker / drag-drop of images -> base64 data URLs, thumbnail previews with remove, forwarded on chat.send (omitted when none). Image-only sends allowed; steering (chat.queue) never forwards images. - ModelSelector: vision badge (isVisionModel) marks vision-capable models; indicator shows native-vision vs vision-handoff hint. Store wiring: ChatStore.send + AppStore.send + App.svelte handleSend thread images through; chat.send still omits cwd (only images added). Verification: svelte-check 0/0; vitest 901/901 (run twice, +34 new); biome clean; vite build OK. See backend-handoff.md §2j. Not merged or pushed. --- src/core/chunks/reducer.test.ts | 153 ++++++++++++++++++++++++++++++++++++++++ src/core/chunks/reducer.ts | 153 ++++++++++++++++++++++++++++++++-------- 2 files changed, 278 insertions(+), 28 deletions(-) (limited to 'src/core/chunks') diff --git a/src/core/chunks/reducer.test.ts b/src/core/chunks/reducer.test.ts index 8a2e1b7..058552e 100644 --- a/src/core/chunks/reducer.test.ts +++ b/src/core/chunks/reducer.test.ts @@ -1,4 +1,5 @@ import type { + ImageInput, StepId, StoredChunk, TurnDoneEvent, @@ -869,3 +870,155 @@ describe("appendUserMessage", () => { expect(s.provisional[1]?.chunk).toEqual({ type: "text", text: "user msg" }); }); }); + +const PNG = "data:image/png;base64,AAAA"; +const JPG = "data:image/jpeg;base64,BBBB"; + +describe("appendUserMessage — images (vision handoff)", () => { + it("echoes text then image chunks in order", () => { + const images: ImageInput[] = [ + { url: PNG, mimeType: "image/png" }, + { url: JPG, mimeType: "image/jpeg" }, + ]; + let s = initialState(); + s = appendUserMessage(s, "what's this?", images); + const chunks = selectChunks(s); + expect(chunks).toHaveLength(3); + expect(chunks[0]?.role).toBe("user"); + expect(chunks[0]?.chunk).toEqual({ type: "text", text: "what's this?" }); + expect(chunks[1]?.chunk).toEqual({ type: "image", url: PNG, mimeType: "image/png" }); + expect(chunks[2]?.chunk).toEqual({ type: "image", url: JPG, mimeType: "image/jpeg" }); + expect(chunks.every((c) => c.provisional && c.seq === null)).toBe(true); + }); + + it("omits mimeType when not provided", () => { + let s = initialState(); + s = appendUserMessage(s, "look", [{ url: PNG }]); + const img = selectChunks(s)[1]?.chunk; + expect(img).toEqual({ type: "image", url: PNG }); + expect(img).not.toHaveProperty("mimeType"); + }); + + it("echoes images-only when text is empty (no text chunk)", () => { + let s = initialState(); + s = appendUserMessage(s, "", [{ url: PNG, mimeType: "image/png" }]); + const chunks = selectChunks(s); + expect(chunks).toHaveLength(1); + expect(chunks[0]?.chunk.type).toBe("image"); + }); + + it("echoes nothing for empty text + empty images", () => { + let s = initialState(); + s = foldEvent(s, turnStart("t1")); + s = foldEvent(s, textDelta("t1", "partial")); + s = appendUserMessage(s, "", []); + // Defensive flush still happened; no user chunk added. + const users = selectChunks(s).filter((c) => c.role === "user"); + expect(users).toHaveLength(0); + expect(s.accumulating).toBeNull(); + }); + + it("skips images with an empty url", () => { + let s = initialState(); + s = appendUserMessage(s, "hi", [ + { url: "", mimeType: "image/png" }, + { url: PNG, mimeType: "image/png" }, + ]); + const chunks = selectChunks(s); + expect(chunks).toHaveLength(2); // text + the one valid image + expect(chunks[1]?.chunk.type).toBe("image"); + }); + + it("groups text + images into one user ChatMessage", () => { + let s = initialState(); + s = appendUserMessage(s, "see this", [{ url: PNG }]); + const msgs = selectMessages(s); + expect(msgs).toHaveLength(1); + expect(msgs[0]?.role).toBe("user"); + expect(msgs[0]?.chunks).toHaveLength(2); + }); +}); + +describe("foldEvent — user-message dedups against a text+image echo", () => { + const userMessage = (text: string): TurnInputEvent => ({ + type: "user-message", + conversationId: "c1", + turnId: "t1", + text, + }); + + it("does not duplicate the text when images follow it in the echo", () => { + let s = initialState(); + s = appendUserMessage(s, "hi", [{ url: PNG, mimeType: "image/png" }]); + expect(selectChunks(s)).toHaveLength(2); // text + image + s = foldEvent(s, userMessage("hi")); + const users = selectChunks(s).filter((c) => c.role === "user"); + expect(users).toHaveLength(2); // unchanged — no duplicate text + expect(users[0]?.chunk.type).toBe("text"); + expect(users[1]?.chunk.type).toBe("image"); + expect(s.generating).toBe(true); + }); + + it("still appends when the echoed text differs", () => { + let s = initialState(); + s = appendUserMessage(s, "first", [{ url: PNG }]); + s = foldEvent(s, userMessage("second")); + const users = selectChunks(s).filter((c) => c.role === "user"); + expect(users.filter((c) => c.chunk.type === "text")).toHaveLength(2); + }); +}); + +describe("applyHistory — multi-chunk image echo is superseded by committed", () => { + it("drops the provisional [text, image] echo when committed arrives during generation", () => { + let s = initialState(); + s = appendUserMessage(s, "hi", [{ url: PNG, mimeType: "image/png" }]); + s = foldEvent(s, turnStart("t1")); + expect(s.provisional).toHaveLength(2); + + s = applyHistory(s, [ + storedChunk(1, "user", { type: "text", text: "hi" }), + storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }), + ]); + expect(s.provisional).toEqual([]); + expect(s.committed).toHaveLength(2); + expect(s.committed[0]?.chunk).toEqual({ type: "text", text: "hi" }); + expect(s.committed[1]?.chunk).toEqual({ type: "image", url: PNG, mimeType: "image/png" }); + }); + + it("keeps the echo when committed is only a partial match (img not yet persisted)", () => { + let s = initialState(); + s = appendUserMessage(s, "hi", [ + { url: PNG, mimeType: "image/png" }, + { url: JPG, mimeType: "image/jpeg" }, + ]); + s = foldEvent(s, turnStart("t1")); + // Only the text + first image have been persisted so far. + s = applyHistory(s, [ + storedChunk(1, "user", { type: "text", text: "hi" }), + storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }), + ]); + // Echo is [text, img1, img2]; committed run is [text, img1] — not a full + // match (echo is longer) → keep the echo (turn-seal will drop it wholesale). + expect(s.provisional.length).toBeGreaterThan(0); + }); + + it("renders committed image chunks from history (a non-vision transcription turn)", () => { + // The server persists the original image chunk AND the transcription text + // in the SAME user message: render both (image, then analysis text). + let s = initialState(); + s = applyHistory(s, [ + storedChunk(1, "user", { type: "text", text: "describe this" }), + storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }), + storedChunk(3, "user", { + type: "text", + text: "[Image analysis (via kimi/k2)]: a red square", + }), + storedChunk(4, "assistant", { type: "text", text: "the square is red" }), + ]); + const msgs = selectMessages(s); + expect(msgs).toHaveLength(2); // one user message (3 chunks) + one assistant + expect(msgs[0]?.role).toBe("user"); + expect(msgs[0]?.chunks).toHaveLength(3); + expect(msgs[0]?.chunks[1]).toEqual({ type: "image", url: PNG, mimeType: "image/png" }); + }); +}); diff --git a/src/core/chunks/reducer.ts b/src/core/chunks/reducer.ts index 2152de3..64e41b9 100644 --- a/src/core/chunks/reducer.ts +++ b/src/core/chunks/reducer.ts @@ -1,4 +1,5 @@ -import type { AgentEvent, Chunk, StoredChunk } from "@dispatch/wire"; +import type { AgentEvent, Chunk, ImageInput, Role, StoredChunk } from "@dispatch/wire"; +import { assertChunkExhaustive } from "../wire/conformance"; import type { AccumulatingChunk, ProvisionalChunk, TranscriptState } from "./types"; /** The initial empty transcript state. */ @@ -42,6 +43,63 @@ function flushAccumulating( return [...provisional, { role: "assistant", chunk }]; } +/** + * Content equality for two chunks of the SAME role (used to de-dup an + * optimistic echo against the authoritative committed version). Compares the + * discriminating payload only — `stepId` (generation provenance) is ignored + * since it is absent on provisional echoes but present on committed tool chunks. + */ +function chunkContentEquals(a: Chunk, b: Chunk): boolean { + if (a.type !== b.type) return false; + switch (a.type) { + case "text": { + const o = b as Extract; + return a.text === o.text; + } + case "thinking": { + const o = b as Extract; + return a.text === o.text; + } + case "image": { + const o = b as Extract; + return a.url === o.url; + } + case "error": { + const o = b as Extract; + return a.message === o.message && a.code === o.code; + } + case "system": { + const o = b as Extract; + return a.text === o.text; + } + case "tool-call": { + const o = b as Extract; + return a.toolCallId === o.toolCallId && a.toolName === o.toolName; + } + case "tool-result": { + const o = b as Extract; + return a.toolCallId === o.toolCallId && a.toolName === o.toolName && a.isError === o.isError; + } + default: + return assertChunkExhaustive(a) === assertChunkExhaustive(b); + } +} + +/** + * The trailing run of consecutive same-role provisional chunks at the end of + * `provisional` (the optimistic echo of one message). Returns the start/end + * indices `[start, end)` into `provisional` (empty if none). + */ +function trailingRun( + provisional: readonly ProvisionalChunk[], + role: Role, +): readonly ProvisionalChunk[] { + const end = provisional.length; + let start = end; + while (start > 0 && provisional[start - 1]?.role === role) start--; + return provisional.slice(start, end); +} + /** * Merge authoritative seq-keyed chunks into the committed history. * Dedupes by seq (new wins), keeps seq-monotonic order, idempotent. @@ -78,21 +136,30 @@ export function applyHistory( // During generation: if new committed chunks arrived, the provisional // array may contain duplicates — the optimistic echo from `appendUserMessage` - // is now backed by a committed chunk (CR-6: user message persisted at turn - // start). Remove provisional chunks that match the last committed chunk - // (role + chunk content), keeping only the accumulating (streaming) chunk. + // is now backed by committed chunks (CR-6: user message persisted at turn + // start). A user message may be multi-chunk (`[text, image, image, …]`), so + // match the trailing provisional user-run against the trailing committed + // user-run by content equality and drop the whole echo when it is fully + // backed. Leaves the accumulating (streaming) chunk untouched. if (addedNew && state.generating && state.provisional.length > 0) { - const lastCommitted = committed[committed.length - 1]; - if (lastCommitted !== undefined) { - const provisional = state.provisional.filter((p) => { - if (p.role !== lastCommitted.role) return true; - if (p.chunk.type !== lastCommitted.chunk.type) return true; - if (p.chunk.type === "text" && lastCommitted.chunk.type === "text") { - return p.chunk.text !== lastCommitted.chunk.text; - } - return true; - }); - return { ...state, committed, provisional, accumulating: state.accumulating }; + const provRun = trailingRun(state.provisional, "user"); + if (provRun.length > 0) { + const commRun = trailingRun(committed, "user"); + // Drop the provisional user echo iff every echoed chunk is content-equal + // to the corresponding committed chunk (the server persisted the same + // message). A partial match (echo longer than committed) keeps the echo + // — the not-yet-committed tail stays until the turn seals. + const fullyBacked = + commRun.length >= provRun.length && + provRun.every((p, i) => { + const c = commRun[i]; + return c !== undefined && chunkContentEquals(p.chunk, c.chunk); + }); + if (fullyBacked) { + const dropStart = state.provisional.length - provRun.length; + const provisional = state.provisional.slice(0, dropStart); + return { ...state, committed, provisional, accumulating: state.accumulating }; + } } } @@ -138,16 +205,18 @@ function reduceEvent(state: TranscriptState, event: AgentEvent): TranscriptState // The turn's USER prompt, surfaced on the event stream (backend CR-3) so a // WATCHER/late-joiner renders it mid-turn instead of waiting for seal. The // SENDER already echoed its own prompt optimistically (`appendUserMessage`), - // so DE-DUP: skip if the trailing provisional chunk is already an identical - // user text chunk. A pure watcher has no such echo → it appends and renders. + // so DE-DUP: skip if the trailing provisional USER run already contains an + // identical user text chunk. The echo may be multi-chunk (`[text, image, + // image, …]` — `appendUserMessage` appends the text first, then images), so + // we scan the whole trailing user run, not just the last chunk (which would + // be an image when images were pasted). A pure watcher has no such echo → + // it appends and renders. The `user-message` event carries ONLY text (never + // images — images arrive via history/loadSince); an images-only send (empty + // text) emits no `user-message` and is not de-duped here. if (event.text.length === 0) return state; - const last = state.provisional[state.provisional.length - 1]; - if ( - last !== undefined && - last.role === "user" && - last.chunk.type === "text" && - last.chunk.text === event.text - ) { + const run = trailingRun(state.provisional, "user"); + const alreadyEchoed = run.some((p) => p.chunk.type === "text" && p.chunk.text === event.text); + if (alreadyEchoed) { return { ...state, generating: true }; } const provisional = flushAccumulating(state.provisional, state.accumulating); @@ -335,15 +404,43 @@ export function foldEvent(state: TranscriptState, event: AgentEvent): Transcript /** * Optimistically append a user message to the provisional list. * Flushes any in-progress accumulating chunk first (defensively). - * The provisional user chunk is superseded when applyHistory receives + * The provisional user chunks are superseded when applyHistory receives * the authoritative committed chunks after a turn seals. + * + * When `images` are provided, they are appended AFTER the text chunk (in + * order) as `image` chunks — matching the server's persisted layout + * (`[text, image, image, …]` in one user message). A text chunk is only + * appended when `text` is non-empty (an images-only send echoes just the + * images). The `user-message` event carries only text (never images), so its + * de-dup scans the trailing user run for the echoed text rather than just the + * last chunk. */ -export function appendUserMessage(state: TranscriptState, text: string): TranscriptState { +export function appendUserMessage( + state: TranscriptState, + text: string, + images?: readonly ImageInput[], +): TranscriptState { const provisional = flushAccumulating(state.provisional, state.accumulating); - const userChunk: Chunk = { type: "text", text }; + const userChunks: Chunk[] = []; + if (text.length > 0) userChunks.push({ type: "text", text }); + if (images !== undefined) { + for (const img of images) { + if (img.url.length === 0) continue; + const chunk: Chunk = + img.mimeType !== undefined + ? { type: "image", url: img.url, mimeType: img.mimeType } + : { type: "image", url: img.url }; + userChunks.push(chunk); + } + } + if (userChunks.length === 0) { + // Nothing to echo (empty text + no images) — leave state unchanged but + // still flush any accumulating chunk defensively. + return { ...state, provisional, accumulating: null }; + } return { ...state, - provisional: [...provisional, { role: "user", chunk: userChunk }], + provisional: [...provisional, ...userChunks.map((chunk) => ({ role: "user", chunk }) as const)], accumulating: null, }; } -- cgit v1.2.3 From f5dc22f7c14d6c0dd4bcedee5a85b21ecd294aed Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 20:20:02 +0900 Subject: feat(vision): resolve persisted image URLs against the API base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Images are now stored on disk under tmp (not SQLite) and served via GET /images/:conversationId/:imageId. Persisted ImageChunk.url is a compact relative HTTP path (/images//.png) instead of a base64 data URL. No wire/transport-contract type change (behavior only) — re-mirrored the delta notes. - New pure resolveImageUrl(url, apiBase) helper (core/chunks/image-url.ts, +8 tests): data/absolute URLs pass through; relative paths are prepended with the API base (no double slash; empty base -> root-relative). Exported from core/chunks + re-exported from features/chat. - ChatView: new apiBaseUrl prop; uses resolveImageUrl. The optimistic echo's data URL passes through; persisted relative paths resolve against the base. +3 tests. - AppStore exposes httpBase (getter); App.svelte passes apiBaseUrl into ChatView and the heartbeat RunModal (also renders image chunks). Verification: svelte-check 0/0; vitest 959/959 (run twice, +11); biome clean; vite build OK. See backend-handoff.md §2j-update-2. Not merged or pushed. --- .dispatch/transport-contract.reference.md | 8 +++++ .dispatch/wire.reference.md | 28 ++++++++++++---- GLOSSARY.md | 2 +- backend-handoff.md | 53 +++++++++++++++++++++++++++---- src/app/App.svelte | 2 ++ src/app/store.svelte.ts | 9 ++++++ src/core/chunks/image-url.test.ts | 46 +++++++++++++++++++++++++++ src/core/chunks/image-url.ts | 35 ++++++++++++++++++++ src/core/chunks/index.ts | 1 + src/features/chat/index.ts | 2 +- src/features/chat/ui.test.ts | 52 ++++++++++++++++++++++++++++++ src/features/chat/ui/ChatView.svelte | 19 ++++++++--- src/features/heartbeat/ui/RunModal.svelte | 7 ++++ 13 files changed, 245 insertions(+), 19 deletions(-) create mode 100644 src/core/chunks/image-url.test.ts create mode 100644 src/core/chunks/image-url.ts (limited to 'src/core/chunks') diff --git a/.dispatch/transport-contract.reference.md b/.dispatch/transport-contract.reference.md index 047f050..182279a 100644 --- a/.dispatch/transport-contract.reference.md +++ b/.dispatch/transport-contract.reference.md @@ -27,6 +27,14 @@ > `PUT /settings/vision` ← `SetVisionSettingsRequest` (partial: `imageLimit?` non-negative int, 0 = disable > compaction; `compactionModel?` `/` or null = auto). See `backend-handoff.md` §2j. > +> **2026-06-26 update (image storage — NO type change, behavior only):** persisted `ImageChunk.url`s are now +> compact relative HTTP paths (`/images//.png`) served by the new +> `GET /images/:conversationId/:imageId` endpoint (raw image bytes + correct Content-Type) — NOT base64 data +> URLs (images are stored on disk under tmp, not in the SQLite store). `ChatRequest.images` (`ImageInput.url`) +> is UNCHANGED — clients still send data URLs; the backend saves them to tmp and returns compact paths in +> the persisted chunks. A client resolves a relative `url` against its API base (`resolveImageUrl`); the +> optimistic echo's data URL and any absolute URL pass through. See `backend-handoff.md` §2j. +> > **2026-06-25 delta (SSH handoff #2 — ADDITIVE to `transport-contract@0.22.0`, NO version bump):** adds the > computer HTTP API types: `ComputerListResponse` (`GET /computers`), `ComputerResponse` (`GET /computers/:alias`), > `ComputerStatusResponse` (`GET /computers/:alias/status`), `TestComputerResponse` (`POST /computers/:alias/test`), diff --git a/.dispatch/wire.reference.md b/.dispatch/wire.reference.md index c40fe81..75adac5 100644 --- a/.dispatch/wire.reference.md +++ b/.dispatch/wire.reference.md @@ -14,6 +14,15 @@ > — the orchestrator's vision handoff transcribes each to a text description (persisted as a separate > `text` chunk in the SAME user message). See `backend-handoff.md` §2j. > +> **2026-06-26 update (image storage — NO type change, behavior only):** `ImageChunk.url` for PERSISTED +> chunks is now a compact relative HTTP path (`/images//.png`) served by the backend's +> new `GET /images/:conversationId/:imageId` endpoint (raw bytes + correct Content-Type), NOT a base64 data +> URL — images are stored on disk under tmp, not in the SQLite conversation store (keeps payloads small). +> `ImageInput.url` (what a client SENDS on `ChatRequest.images`) is UNCHANGED — still a data URL or +> `http(s)://` URL; the backend saves it to tmp and returns the compact path in the persisted chunk. A client +> resolves a relative `url` against its API base (`resolveImageUrl`); a data URL (the optimistic echo) or an +> absolute URL passes through unchanged. See `backend-handoff.md` §2j. +> > **2026-06-23 delta (workspaces handoff — package bumped `0.11.0` → `0.12.0`, ADDITIVE):** adds > `Workspace` + `WorkspaceEntry` (a list entry with a conversation count) and a required > `workspaceId: string` on `ConversationMeta` (`"default"` for legacy/unspecified conversations). A @@ -156,12 +165,17 @@ export interface SystemChunk { /** * An image attached to a message (e.g. a user-pasted screenshot or pasted * photo). Carries a `url` that is EITHER a base64 data URL - * (`data:image/png;base64,…`) OR an `http(s)://` URL. Vision-capable models - * receive it natively (the provider serializes it to its image-content - * format); non-vision models never see it directly — the orchestrator's - * **vision handoff** transcribes it to a text description (via a - * vision-capable model) and feeds that text instead, so a text-only model can - * still reason about the image's contents. + * (`data:image/png;base64,…`) OR an `http(s)://` URL OR — for PERSISTED chunks + * (history/replay) — a compact relative HTTP path (`/images// + * .png`) served by the backend's `GET /images/:conversationId/:imageId` + * endpoint (images are stored on disk under tmp, NOT in the conversation store, + * to keep SQLite payloads small). A client resolves a relative path against its + * API base URL; a data URL (the optimistic echo / a pasted image) or an + * absolute URL is rendered as-is. Vision-capable models receive it natively + * (the provider serializes it to its image-content format); non-vision models + * never see it directly — the orchestrator's **vision handoff** transcribes it + * to a text description (via a vision-capable model) and feeds that text + * instead, so a text-only model can still reason about the image's contents. * * When a transcription was performed, it is persisted as a separate `text` * chunk alongside the `image` chunk in the SAME user message, so the @@ -170,7 +184,7 @@ export interface SystemChunk { */ export interface ImageChunk { readonly type: "image"; - /** Image source: a base64 data URL (`data:image/…;base64,…`) or an `http(s)://` URL. */ + /** Image source: a base64 data URL (`data:image/…;base64,…`), an `http(s)://` URL, or a compact relative path (`/images//.png`) for persisted chunks. */ readonly url: string; /** * Optional MIME type of the image (e.g. `"image/png"`). Inferred from the diff --git a/GLOSSARY.md b/GLOSSARY.md index 3973fe1..14b81d1 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -26,7 +26,7 @@ | **steering** | A user message injected into an in-flight turn at the tool-result boundary (drawn from the **message queue**): the model sees it alongside the tool results and may adjust course. Emitted on the chat stream as a `steering` `AgentEvent` (`TurnSteeringEvent`); the queue surface clears on drain (move, don't duplicate). If the turn ends with a non-empty queue (no tool call fired), the queue carries into a NEW turn as its opening prompt (no `steering` event). `wire@0.8.0`. | mid-turn injection, course correction, interruption | | **computer** | A remote SSH target discovered from the system's `~/.ssh/config` — a read-only VIEW, NOT an editable entity (no CRUD store; to add one the user edits `~/.ssh/config`). Backend-canonical (`wire@0.12.0`, additive). On the wire as `Computer` (`{ alias, hostName, port, user, identityFile, knownHost }`) + `ComputerEntry extends Computer` (adds `usageCount`, for `GET /computers`). `alias` IS the **computerId** — the string persisted per-conversation / per-workspace (the computer analog of `cwd`). Resolution is SERVER-owned (never re-implement): per-conversation `computerId` → `workspace.defaultComputerId` → `null`/local. USER-facing only: a tool-execution target forwarded to tools, NEVER part of the model prompt (does not affect prompt caching); the agent never sees it. HTTP API (`GET /computers`, `GET`/`PUT /conversations/:id/computer`, `PUT /workspaces/:id/default-computer`, `GET /computers/:alias/status`, `POST /computers/:alias/test`) consumed in handoff #2. | ssh host, remote host, server, connection target | | **computerId** | The string id of a **computer** — an SSH config `Host` alias users select. Persisted per-conversation and per-workspace (the computer analog of `cwd`/`workspaceId`). `null` means local (no SSH; today's behavior). On `Workspace` as the REQUIRED `defaultComputerId: string \| null` (null = local / no SSH; the computer analog of `defaultCwd`); per-conversation persistence via `GET`/`PUT`/`DELETE /conversations/:id/computer`. `chat.send` need not send it (resolved server-side from the persisted per-conversation value in the MVP). | ssh alias, host id, remote id | -| **image chunk** | An `ImageChunk` (`{ type: "image", url, mimeType? }`) — a NEW `Chunk` variant for an image attached to a message (a user-pasted screenshot/photo). `url` is a base64 data URL (`data:image/…;base64,…`) OR an `http(s)://` URL. Backend-canonical (`wire@0.12.0`, additive). A user message may be multi-chunk (`[text, image, image, …]` in order). On the wire as `ImageChunk` (persisted) + `ImageInput` (what `ChatRequest.images` carries; the orchestrator converts each into an `ImageChunk`). | picture, photo attachment, screenshot chunk | +| **image chunk** | An `ImageChunk` (`{ type: "image", url, mimeType? }`) — a `Chunk` variant for an image attached to a message (a user-pasted screenshot/photo). `url` is a base64 data URL (`data:image/…;base64,…`), an `http(s)://` URL, or — for PERSISTED chunks — a compact relative HTTP path (`/images//.png`) served by `GET /images/:conversationId/:imageId` (images are stored on disk under tmp, not in the SQLite store). Backend-canonical (`wire@0.12.0`, additive). A user message may be multi-chunk (`[text, image, image, …]` in order). On the wire as `ImageChunk` (persisted) + `ImageInput` (what `ChatRequest.images` carries — still a data URL; the orchestrator saves it to tmp and returns the compact path). A client resolves a relative `url` against its API base (`resolveImageUrl`); a data URL (the optimistic echo) or absolute URL passes through. | picture, photo attachment, screenshot chunk | | **vision** (capability) | Whether a model can natively accept image input (multimodal). On the wire as `ModelMetadata.vision?: boolean` (`GET /models` `modelInfo[name].vision`). `true` (e.g. any `kimi/*` model) → image chunks are passed through to the provider natively. Absent/`false` (e.g. `umans/glm-5.2`) → the server's **vision handoff** gives the model a numbered placeholder + the `consult_vision` tool. The FE shows a vision badge in the model picker; it does NOT decide handoff (server-owned). | multimodal, image support | | **vision handoff** | The server-owned mechanism by which a NON-vision model still reasons about a pasted image. A non-vision model gets a NUMBERED PLACEHOLDER text chunk alongside the persisted `image` chunk, then calls the `consult_vision` tool (which opens a NEW conversation tab with a vision-capable model, attaches the image + question, and returns the vision model's answer). The persisted user message keeps the original `image` chunk (the FE renders it) AND the placeholder text. When a vision-capable model has more than `imageLimit` images in history, image **compaction** transcribes the oldest to `[Compacted image]: ` text chunks (the `image` chunk stays for rendering). All these are regular `text` chunks — the FE renders them as-is. Distinct from a vision-capable model, which receives the image natively. | image transcription, vision relay | | **consult_vision** | A tool available to all models that defers image analysis to a vision-capable model: it opens a NEW conversation tab (with a vision model, e.g. Kimi), attaches the image (by `imageIds` from a pasted placeholder, or by `path` from disk) + a `question`, and returns the conversation id + the vision model's answer (suggesting the dispatch CLI for follow-ups). Replaces the former `read_image` tool. Rendered generically like any tool call/result (by `toolName`). | read_image (former), vision tool | diff --git a/backend-handoff.md b/backend-handoff.md index 9338a5a..5a476be 100644 --- a/backend-handoff.md +++ b/backend-handoff.md @@ -5,12 +5,11 @@ > **From:** dispatch-web orchestrator · **To:** `../backend` orchestrator · **Courier:** the user. > `lsp` does NOT span the repos (AGENTS.md § Backend seam) — every cross-repo ask flows through here. -_Last updated: 2026-06-26 (§2j UPDATED — consult_vision tool + vision settings API: `read_image` is REPLACED by -`consult_vision` (non-vision models now get numbered placeholder text chunks, not auto-transcriptions); NEW global -`GET`/`PUT /settings/vision` (`VisionSettingsResponse`/`SetVisionSettingsRequest` — `imageLimit` + `compactionModel`); -image compaction transcribes old images to `[Compacted image]: …` text chunks (all regular text — render as-is). New -`vision` feature library + "Vision" sidebar view (imageLimit input + compactionModel dropdown of vision-capable models + -"Auto"). typecheck 0/0, 948 tests green (+47), biome clean, build OK. §2i unchanged.)_ +_Last updated: 2026-06-26 (§2j UPDATED — Image storage: persisted `ImageChunk.url`s are now compact relative HTTP +paths (`/images//.png`) served by `GET /images/:conversationId/:imageId` (images stored on disk under tmp, +not SQLite). FE resolves relative urls against the API base via a new pure `resolveImageUrl` helper; the optimistic +echo's data URL passes through unchanged; `ChatRequest.images` (send) is unchanged. typecheck 0/0, 959 tests green +(+11), biome clean, build OK. §2i unchanged.)_ **FE is current on `ui-contract@0.2.0` / `transport-contract@0.22.0` / `wire@0.12.0`.** Open asks: **CR-9** (`system:os` should detect WSL + include Linux distro — backend behavior change, no contract bump). The SSH-divergence (§2d) is RESOLVED. @@ -854,6 +853,48 @@ reload; set compactionModel to a vision model → confirm the dropdown reflects non-vision model → confirm the `[Image N attached — call consult_vision…]` placeholder renders (not an auto-transcription); trigger a `consult_vision` tool call → confirm it renders like a tool. +### 2j-update-2. Image storage (tmp, not SQLite) → **CONSUMED ✅ (backend updated; FE built + verified)** + +A follow-up to §2j. The backend no longer persists images as base64 data URLs in the conversation store — +they are saved to a tmp directory and served via HTTP. The `ImageChunk.url` field's FORMAT changed (the +TYPE is unchanged — still `string`); `GET /images/:conversationId/:imageId` is a new endpoint serving raw +bytes + the correct Content-Type. **NO wire/transport-contract type change** (behavior only); re-mirrored +the delta notes in `.dispatch/wire.reference.md` + `.dispatch/transport-contract.reference.md`. + +**What changed (backend):** +- **BEFORE:** `ImageChunk.url` was a base64 data URL (`data:image/png;base64,…`). +- **NOW:** `ImageChunk.url` for PERSISTED chunks (history/replay) is a compact relative HTTP path + (`/images//.png`), served by `GET /images/:conversationId/:imageId` (raw image + bytes + Content-Type). Images live on disk under tmp, NOT in the SQLite conversation store (keeps + payloads small). +- **`ChatRequest.images` (`ImageInput.url`) is UNCHANGED** — the FE still sends data URLs; the backend + saves them to tmp and returns compact paths in the persisted chunks. +- **Optimistic echo:** the FE's provisional echo still uses the data URL it sent (immediate render); + when the persisted chunk arrives (via `loadSince`/`syncTail`/event stream), it carries the compact path + and the FE switches to rendering via the HTTP endpoint. +- The vision settings API, `consult_vision`, and image compaction are UNCHANGED (compaction resolves + compact URLs internally). + +**FE (DONE + verified):** +- **New pure helper `resolveImageUrl(url, apiBase)`** (`core/chunks/image-url.ts`, +8 tests, exported from + `core/chunks` + re-exported from `features/chat`): a `data:` URL or an `http(s)://` URL passes through + unchanged; a relative path (`/images/…`) is prepended with the API base (no double slash; an empty base + yields a root-relative path a browser resolves against its origin). Pure — zero DOM/Svelte. +- **`ChatView.svelte`:** new `apiBaseUrl` prop (default `""`); the `` now uses + `resolveImageUrl(rendered.chunk.url, apiBaseUrl)`. The optimistic echo's data URL + any absolute URL + pass through; persisted relative paths resolve against the base. +3 ChatView tests (relative-path + resolution, data-URL pass-through with a base set, root-relative when no base). +- **Store + wiring:** `httpBase` (the resolved HTTP API base) is now exposed as a getter on `AppStore`; + `App.svelte` passes `apiBaseUrl={store.httpBase}` to `ChatView` AND to the heartbeat `RunModal` (its + `ChatView` also renders image chunks — added an `apiBaseUrl` prop there, threaded from `App.svelte`). + +**Verification:** `svelte-check` 0/0; vitest **959/959** (run TWICE — no cross-test pollution; +11 since the +prior commit: 8 `resolveImageUrl`, 3 ChatView resolution), biome clean, `vite build` succeeds. +**Live probe NOT run** (backend not reachable headless). To confirm end-to-end: paste an image with a +vision model, send, confirm the image renders immediately (data-URL echo) AND continues to render after +the turn seals (the persisted compact-path `/images/…` resolved against `httpBase`); reload the +conversation → confirm the persisted image renders from the `/images/…` endpoint (not a data URL). + --- ## 3. Likely NEXT backend asks (heads-up, not yet requested) diff --git a/src/app/App.svelte b/src/app/App.svelte index f41d7ba..7d70f67 100644 --- a/src/app/App.svelte +++ b/src/app/App.svelte @@ -516,6 +516,7 @@ onShowEarlier={handleShowEarlier} thinkingKeyBase={store.activeChat.thinkingKeyBase} providerRetry={store.activeChat.providerRetry} + apiBaseUrl={store.httpBase} /> {/key} @@ -610,6 +611,7 @@ closeChat={closeRunChat} stopRun={stopHeartbeatRun} onClose={() => (heartbeatRun = null)} + apiBaseUrl={store.httpBase} /> {/key} {/if} diff --git a/src/app/store.svelte.ts b/src/app/store.svelte.ts index 554d5e0..8353820 100644 --- a/src/app/store.svelte.ts +++ b/src/app/store.svelte.ts @@ -157,6 +157,12 @@ export interface AppStore { readonly activeConversationId: string | null; /** The workspace currently in view (URL slug); tabs are filtered to it. */ readonly activeWorkspaceId: string; + /** + * The resolved HTTP API base URL (e.g. `http://localhost:24203`). Used to + * resolve relative image URLs served by the backend (`/images/…`) into + * absolute URLs for ``. + */ + readonly httpBase: string; readonly activeChat: ChatStore; readonly models: readonly string[]; /** Per-model metadata (contextWindow, etc.) from `GET /models`. */ @@ -1127,6 +1133,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { get activeWorkspaceId(): string { return activeWorkspaceId; }, + get httpBase(): string { + return httpBase; + }, setActiveWorkspace(workspaceId: string): void { activeWorkspaceId = workspaceId; // Reset to a fresh draft scoped to the new workspace so a new chat is diff --git a/src/core/chunks/image-url.test.ts b/src/core/chunks/image-url.test.ts new file mode 100644 index 0000000..8a79c09 --- /dev/null +++ b/src/core/chunks/image-url.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { resolveImageUrl } from "./image-url"; + +const BASE = "http://localhost:24203"; + +describe("resolveImageUrl", () => { + it("returns a data URL as-is (the optimistic echo / a pasted image)", () => { + const dataUrl = "data:image/png;base64,iVBORw0KGgo="; + expect(resolveImageUrl(dataUrl, BASE)).toBe(dataUrl); + }); + + it("returns an absolute http URL as-is", () => { + const abs = "https://example.com/img.png"; + expect(resolveImageUrl(abs, BASE)).toBe(abs); + }); + + it("prepends the api base to a relative /images/ path", () => { + expect(resolveImageUrl("/images/conv-123/abc-456.png", BASE)).toBe( + "http://localhost:24203/images/conv-123/abc-456.png", + ); + }); + + it("does not double the slash when the base has a trailing slash", () => { + expect(resolveImageUrl("/images/c/x.png", "http://localhost:24203/")).toBe( + "http://localhost:24203/images/c/x.png", + ); + }); + + it("adds a leading slash to a path-relative url without one", () => { + expect(resolveImageUrl("images/c/x.png", BASE)).toBe("http://localhost:24203/images/c/x.png"); + }); + + it("returns the relative path as-is when apiBase is empty (root-relative)", () => { + // A browser resolves a root-relative `/images/…` against the document origin. + expect(resolveImageUrl("/images/c/x.png", "")).toBe("/images/c/x.png"); + }); + + it("handles a relative path with an empty apiBase (path-relative without slash)", () => { + expect(resolveImageUrl("images/c/x.png", "")).toBe("/images/c/x.png"); + }); + + it("returns a data URL as-is even with an empty apiBase", () => { + const dataUrl = "data:image/jpeg;base64,AAAA"; + expect(resolveImageUrl(dataUrl, "")).toBe(dataUrl); + }); +}); diff --git a/src/core/chunks/image-url.ts b/src/core/chunks/image-url.ts new file mode 100644 index 0000000..e5ec756 --- /dev/null +++ b/src/core/chunks/image-url.ts @@ -0,0 +1,35 @@ +/** + * Resolve an `ImageChunk.url` into a renderable `` value. + * + * Persisted image chunks now carry a COMPACT HTTP path + * (`/images//.png`) served by the backend — NOT a base64 + * data URL (images are stored on disk under tmp, not in the conversation store, + * to keep SQLite payloads small). The optimistic echo (what the FE just sent in + * `ChatRequest.images`) still carries a data URL, and a chunk could also carry + * an absolute `http(s)://` URL, so the resolution is format-aware: + * + * - `data:` URL → returned as-is (the optimistic echo / a pasted data URL). + * - `http(s)://` → returned as-is (an absolute URL already). + * - anything else (a relative path like `/images/…`) → `apiBase` is prepended + * (with no double slash). An empty `apiBase` leaves a root-relative path, + * which a browser resolves against the document origin. + * + * Pure: input → output, zero DOM, zero Svelte. + * + * @param url The chunk's `url` (data URL, absolute, or relative path). + * @param apiBase The HTTP API base URL (e.g. `http://localhost:24203`). + */ +export function resolveImageUrl(url: string, apiBase: string): string { + if (url.startsWith("data:") || url.startsWith("http://") || url.startsWith("https://")) { + return url; + } + // A relative path (e.g. `/images/…`) — normalize to a leading slash and + // prepend the api base. With an empty base this yields a root-relative path + // (a browser resolves `/images/…` against the document origin). + const path = url.startsWith("/") ? url : `/${url}`; + if (apiBase.length === 0) return path; + // Join without a double slash: strip a trailing slash from the base, then + // append the (leading-slash) path verbatim. + const base = apiBase.endsWith("/") ? apiBase.slice(0, -1) : apiBase; + return `${base}${path}`; +} diff --git a/src/core/chunks/index.ts b/src/core/chunks/index.ts index eea2303..bdd6ce4 100644 --- a/src/core/chunks/index.ts +++ b/src/core/chunks/index.ts @@ -1,5 +1,6 @@ export type { RenderGroup, ToolBatchEntry } from "./groups"; export { groupRenderedChunks } from "./groups"; +export { resolveImageUrl } from "./image-url"; export { appendUserMessage, applyHistory, diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts index 8694691..773cb91 100644 --- a/src/features/chat/index.ts +++ b/src/features/chat/index.ts @@ -4,7 +4,7 @@ export type { RenderGroup, ToolBatchEntry, } from "../../core/chunks"; -export { groupRenderedChunks, viewProviderRetry } from "../../core/chunks"; +export { groupRenderedChunks, resolveImageUrl, viewProviderRetry } from "../../core/chunks"; export type { TurnMetricsEntry } from "../../core/metrics"; export { isVisionModel } from "./model-select"; export type { ChatTransport, HistorySync, HistoryWindow, MetricsSync } from "./ports"; diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index b0aa6f0..5f8067d 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -628,6 +628,58 @@ describe("ChatView", () => { expect(img?.getAttribute("loading")).toBe("lazy"); }); + it("resolves a persisted image chunk's relative url against apiBaseUrl", () => { + // Persisted image chunks now carry a compact relative path (`/images/…`) + // served by the backend — prepend the API base to render them. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url: "/images/conv-123/abc-456.png", mimeType: "image/png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe( + "http://localhost:24203/images/conv-123/abc-456.png", + ); + }); + + it("passes a data URL through unchanged even with apiBaseUrl set (optimistic echo)", () => { + // The optimistic echo (what the FE just sent) is still a data URL; it must + // NOT be mangled by the base-URL prepend. + const dataUrl = "data:image/png;base64,iVBOR="; + const chunks: RenderedChunk[] = [ + { seq: null, role: "user", chunk: { type: "image", url: dataUrl }, provisional: true }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe(dataUrl); + }); + + it("leaves a relative image url root-relative when apiBaseUrl is absent", () => { + // No apiBaseUrl → a browser resolves `/images/…` against the document origin. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url: "/images/conv-1/x.png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { props: { chunks } }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe("/images/conv-1/x.png"); + }); + it("renders a multi-chunk user message [text, image] and a transcription text", () => { // A non-vision model: the server persists the original image chunk AND a // transcription text chunk in the SAME user message — render both. diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte index 8081951..cd69071 100644 --- a/src/features/chat/ui/ChatView.svelte +++ b/src/features/chat/ui/ChatView.svelte @@ -1,6 +1,6 @@