diff options
| author | Adam Malczewski <[email protected]> | 2026-06-27 04:18:59 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-27 04:18:59 +0900 |
| commit | 3566a20ebbded754070fce66af48690d1a904879 (patch) | |
| tree | ad4941abdbe685e2d4ae85d9ad3d0b6e9cf82c2d /src/app | |
| parent | 2fa03f8d7410c2b8d6be8e10ad088863e83d7177 (diff) | |
| download | dispatch-web-3566a20ebbded754070fce66af48690d1a904879.tar.gz dispatch-web-3566a20ebbded754070fce66af48690d1a904879.zip | |
feat(vision): image paste + transcript image rendering + vision badge
Vision & vision-handoff frontend (consumes the backend's additive
[email protected] / [email protected] image types — no version bump).
Contracts mirrored:
- .dispatch/wire.reference.md: ImageChunk added to the Chunk union +
ImageChunk/ImageInput interfaces.
- .dispatch/transport-contract.reference.md: ChatRequest.images,
ModelMetadata.vision, + ImageChunk/ImageInput re-exports.
Core (core/chunks):
- conformance: assertChunkExhaustive handles the new 'image' variant
(the guard caught it — its purpose).
- appendUserMessage(state, text, images?) echoes a [text, image, ...]
user run; the user-message event dedup scans the trailing user run
(not just the last chunk) so an image-bearing echo doesn't duplicate
the text; applyHistory's during-gen dedup matches a multi-chunk echo
by content equality (chunkContentEquals + trailingRun helpers).
UI:
- ChatView renders user 'image' chunks as lazy <img> bubbles; a
non-vision model's persisted [image, analysis-text] both render.
read_image tool renders generically (no special-casing).
- Composer: clipboard paste / file picker / drag-drop of images ->
base64 data URLs, thumbnail previews with remove, forwarded on
chat.send (omitted when none). Image-only sends allowed; steering
(chat.queue) never forwards images.
- ModelSelector: vision badge (isVisionModel) marks vision-capable
models; indicator shows native-vision vs vision-handoff hint.
Store wiring: ChatStore.send + AppStore.send + App.svelte handleSend
thread images through; chat.send still omits cwd (only images added).
Verification: svelte-check 0/0; vitest 901/901 (run twice, +34 new);
biome clean; vite build OK. See backend-handoff.md §2j.
Not merged or pushed.
Diffstat (limited to 'src/app')
| -rw-r--r-- | src/app/App.svelte | 13 | ||||
| -rw-r--r-- | src/app/store.svelte.ts | 17 | ||||
| -rw-r--r-- | src/app/store.test.ts | 28 |
3 files changed, 49 insertions, 9 deletions
diff --git a/src/app/App.svelte b/src/app/App.svelte index 09be947..f240a06 100644 --- a/src/app/App.svelte +++ b/src/app/App.svelte @@ -1,5 +1,5 @@ <script lang="ts"> - import type { ReasoningEffort } from "@dispatch/transport-contract"; + import type { ImageInput, ReasoningEffort } from "@dispatch/transport-contract"; import type { InvokeMessage } from "@dispatch/ui-contract"; import { tick } from "svelte"; import Table from "../components/Table.svelte"; @@ -241,8 +241,8 @@ store.invoke(msg.surfaceId, msg.actionId, msg.payload); } - function handleSend(text: string) { - store.send(text); + function handleSend(text: string, images?: readonly ImageInput[]): void { + store.send(text, images); } function handleQueue(text: string) { @@ -587,7 +587,12 @@ {#snippet viewContent(kind: string)} {#if kind === "model"} <div class="flex flex-col gap-3"> - <ModelSelector models={store.models} selected={store.activeModel} onSelect={handleSelectModel} /> + <ModelSelector + models={store.models} + selected={store.activeModel} + onSelect={handleSelectModel} + modelInfo={store.modelInfo} + /> <!-- Keyed on the workspace conversation (active tab OR draft) so the inputs re-mount per conversation — incl. switching between drafts — and can't bleed across tabs. Editable for a draft too (cwd + effort apply from turn 1). --> diff --git a/src/app/store.svelte.ts b/src/app/store.svelte.ts index 78f6ede..41f3a5d 100644 --- a/src/app/store.svelte.ts +++ b/src/app/store.svelte.ts @@ -35,7 +35,7 @@ import type { WarmResponse, } from "@dispatch/transport-contract"; import type { SubscribeMessage, SurfaceServerMessage, SurfaceSpec } from "@dispatch/ui-contract"; -import type { ComputerEntry, ConversationStatus } from "@dispatch/wire"; +import type { ComputerEntry, ConversationStatus, ImageInput } from "@dispatch/wire"; import { untrack } from "svelte"; import { createIdbChunkStore } from "../adapters/idb"; import { createLocalStore } from "../adapters/local-storage"; @@ -160,7 +160,14 @@ export interface AppStore { readonly storage: Storage | undefined; /** The current spec for one surface by id (discovery-by-id), or null if absent. */ surface(surfaceId: string): SurfaceSpec | null; - send(text: string): void; + /** + * Send a user message (start a turn). Forwards any staged `images` + * (`ImageInput[]` — base64 data URLs / https URLs) on the `chat.send` op; + * the server passes them to a vision-capable model natively or transcribes + * them via vision handoff for a non-vision model. Omitted on the wire when + * none are staged. On a draft, promotes to a tab first. + */ + send(text: string, images?: readonly ImageInput[]): void; /** * Enqueue a steering message onto the focused conversation's queue * (`chat.queue` WS op). While a turn is generating, the message is delivered @@ -1149,7 +1156,7 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { return getSurfaceSpec(protocol, surfaceId); }, - send(text: string): void { + send(text: string, images?: readonly ImageInput[]): void { if (tabsStore.activeConversationId === null) { // Draft: promote to tab on first send const conversationId = draftConversationId; @@ -1177,9 +1184,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { void refreshReasoningEffort(); void refreshCompactPercent(); // Now send on the promoted store - chatStores.get(conversationId)?.send(text); + chatStores.get(conversationId)?.send(text, images); } else { - activeChat.send(text); + activeChat.send(text, images); } }, diff --git a/src/app/store.test.ts b/src/app/store.test.ts index 947a9b0..7130c4d 100644 --- a/src/app/store.test.ts +++ b/src/app/store.test.ts @@ -423,6 +423,34 @@ describe("createAppStore", () => { store.dispose(); }); + it("sending from draft forwards staged images on chat.send", () => { + const ws = fakeSocket(); + const store = createAppStore({ + socketFactory: () => ws, + fetchImpl: fakeFetchImpl(), + localStorage: createFakeStorage(), + }); + ws.resolveOpen(); + ws.sent.length = 0; + + const images = [ + { url: "data:image/png;base64,AAAA", mimeType: "image/png" }, + { url: "https://example.com/x.jpg" }, + ]; + store.send("describe these", images); + + const msgs = parseSent(ws); + const chatSend = msgs.find((m) => (m as { type: string }).type === "chat.send") as + | { type: string; message: string; images?: { url: string }[] } + | undefined; + expect(chatSend).toBeTruthy(); + expect(chatSend?.images).toEqual(images); + // The optimistic echo includes the image chunks. + expect(store.activeChat.chunks.some((c) => c.chunk.type === "image")).toBe(true); + + store.dispose(); + }); + it("an incoming chat.delta renders in the transcript", () => { const ws = fakeSocket(); const store = createAppStore({ |
