From 4d762ce9614db118633fb919153d1cccd4965559 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 04:18:59 +0900 Subject: feat(vision): image paste + transcript image rendering + vision badge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vision & vision-handoff frontend (consumes the backend's additive wire@0.12.0 / transport-contract@0.22.0 image types โ€” no version bump). Contracts mirrored: - .dispatch/wire.reference.md: ImageChunk added to the Chunk union + ImageChunk/ImageInput interfaces. - .dispatch/transport-contract.reference.md: ChatRequest.images, ModelMetadata.vision, + ImageChunk/ImageInput re-exports. Core (core/chunks): - conformance: assertChunkExhaustive handles the new 'image' variant (the guard caught it โ€” its purpose). - appendUserMessage(state, text, images?) echoes a [text, image, ...] user run; the user-message event dedup scans the trailing user run (not just the last chunk) so an image-bearing echo doesn't duplicate the text; applyHistory's during-gen dedup matches a multi-chunk echo by content equality (chunkContentEquals + trailingRun helpers). UI: - ChatView renders user 'image' chunks as lazy bubbles; a non-vision model's persisted [image, analysis-text] both render. read_image tool renders generically (no special-casing). - Composer: clipboard paste / file picker / drag-drop of images -> base64 data URLs, thumbnail previews with remove, forwarded on chat.send (omitted when none). Image-only sends allowed; steering (chat.queue) never forwards images. - ModelSelector: vision badge (isVisionModel) marks vision-capable models; indicator shows native-vision vs vision-handoff hint. Store wiring: ChatStore.send + AppStore.send + App.svelte handleSend thread images through; chat.send still omits cwd (only images added). Verification: svelte-check 0/0; vitest 901/901 (run twice, +34 new); biome clean; vite build OK. See backend-handoff.md ยง2j. Not merged or pushed. --- src/features/chat/store.test.ts | 89 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) (limited to 'src/features/chat/store.test.ts') diff --git a/src/features/chat/store.test.ts b/src/features/chat/store.test.ts index c052d1b..8f36994 100644 --- a/src/features/chat/store.test.ts +++ b/src/features/chat/store.test.ts @@ -1,4 +1,4 @@ -import type { AgentEvent, StepId, StoredChunk } from "@dispatch/wire"; +import type { AgentEvent, ImageInput, StepId, StoredChunk } from "@dispatch/wire"; import { describe, expect, it, vi } from "vitest"; import { createChatStore } from "./store.svelte"; import { @@ -144,6 +144,93 @@ describe("createChatStore", () => { store.dispose(); }); + it("send forwards staged images on chat.send and echoes them provisionally", () => { + const transport = createFakeTransport(); + const historySync = createFakeHistorySync(); + const metricsSync = createFakeMetricsSync(); + const cache = createFakeCache(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: historySync.impl, + metricsSync: metricsSync.impl, + cache: cache.impl, + }); + + const images: ImageInput[] = [ + { url: "data:image/png;base64,AAAA", mimeType: "image/png" }, + { url: "https://example.com/cat.jpg" }, + ]; + store.send("look at this", images); + + expect(transport.sent).toHaveLength(1); + const msg = transport.sent[0]; + expect(msg?.type).toBe("chat.send"); + expect(msg?.message).toBe("look at this"); + expect(msg?.images).toEqual(images); + + // Optimistic echo: a text chunk + two image chunks, provisional. + const chunks = store.chunks; + expect(chunks).toHaveLength(3); + expect(chunks[0]?.chunk).toEqual({ type: "text", text: "look at this" }); + expect(chunks[1]?.chunk).toEqual({ type: "image", url: images[0]?.url, mimeType: "image/png" }); + expect(chunks[2]?.chunk).toEqual({ type: "image", url: images[1]?.url }); + + store.dispose(); + }); + + it("send omits images on the wire when none are staged (backward compatible)", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("just text"); + + expect(transport.sent[0]).not.toHaveProperty("images"); + store.dispose(); + }); + + it("send omits images on the wire for an empty array", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("just text", []); + + expect(transport.sent[0]).not.toHaveProperty("images"); + store.dispose(); + }); + + it("send allows an images-only message (empty text)", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("", [{ url: "data:image/png;base64,AAAA", mimeType: "image/png" }]); + + expect(transport.sent[0]?.message).toBe(""); + expect(transport.sent[0]?.images).toHaveLength(1); + // The echo is image-only (no text chunk). + expect(store.chunks).toHaveLength(1); + expect(store.chunks[0]?.chunk.type).toBe("image"); + store.dispose(); + }); + describe("queueMessage (chat.queue โ€” steering)", () => { it("posts a chat.queue with conversationId + text", () => { const transport = createFakeTransport(); -- cgit v1.2.3