summaryrefslogtreecommitdiffhomepage
path: root/src/app
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-27 04:18:59 +0900
committerAdam Malczewski <[email protected]>2026-06-27 04:18:59 +0900
commit3566a20ebbded754070fce66af48690d1a904879 (patch)
treead4941abdbe685e2d4ae85d9ad3d0b6e9cf82c2d /src/app
parent2fa03f8d7410c2b8d6be8e10ad088863e83d7177 (diff)
downloaddispatch-web-3566a20ebbded754070fce66af48690d1a904879.tar.gz
dispatch-web-3566a20ebbded754070fce66af48690d1a904879.zip
feat(vision): image paste + transcript image rendering + vision badge
Vision & vision-handoff frontend (consumes the backend's additive [email protected] / [email protected] image types — no version bump). Contracts mirrored: - .dispatch/wire.reference.md: ImageChunk added to the Chunk union + ImageChunk/ImageInput interfaces. - .dispatch/transport-contract.reference.md: ChatRequest.images, ModelMetadata.vision, + ImageChunk/ImageInput re-exports. Core (core/chunks): - conformance: assertChunkExhaustive handles the new 'image' variant (the guard caught it — its purpose). - appendUserMessage(state, text, images?) echoes a [text, image, ...] user run; the user-message event dedup scans the trailing user run (not just the last chunk) so an image-bearing echo doesn't duplicate the text; applyHistory's during-gen dedup matches a multi-chunk echo by content equality (chunkContentEquals + trailingRun helpers). UI: - ChatView renders user 'image' chunks as lazy <img> bubbles; a non-vision model's persisted [image, analysis-text] both render. read_image tool renders generically (no special-casing). - Composer: clipboard paste / file picker / drag-drop of images -> base64 data URLs, thumbnail previews with remove, forwarded on chat.send (omitted when none). Image-only sends allowed; steering (chat.queue) never forwards images. - ModelSelector: vision badge (isVisionModel) marks vision-capable models; indicator shows native-vision vs vision-handoff hint. Store wiring: ChatStore.send + AppStore.send + App.svelte handleSend thread images through; chat.send still omits cwd (only images added). Verification: svelte-check 0/0; vitest 901/901 (run twice, +34 new); biome clean; vite build OK. See backend-handoff.md §2j. Not merged or pushed.
Diffstat (limited to 'src/app')
-rw-r--r--src/app/App.svelte13
-rw-r--r--src/app/store.svelte.ts17
-rw-r--r--src/app/store.test.ts28
3 files changed, 49 insertions, 9 deletions
diff --git a/src/app/App.svelte b/src/app/App.svelte
index 09be947..f240a06 100644
--- a/src/app/App.svelte
+++ b/src/app/App.svelte
@@ -1,5 +1,5 @@
<script lang="ts">
- import type { ReasoningEffort } from "@dispatch/transport-contract";
+ import type { ImageInput, ReasoningEffort } from "@dispatch/transport-contract";
import type { InvokeMessage } from "@dispatch/ui-contract";
import { tick } from "svelte";
import Table from "../components/Table.svelte";
@@ -241,8 +241,8 @@
store.invoke(msg.surfaceId, msg.actionId, msg.payload);
}
- function handleSend(text: string) {
- store.send(text);
+ function handleSend(text: string, images?: readonly ImageInput[]): void {
+ store.send(text, images);
}
function handleQueue(text: string) {
@@ -587,7 +587,12 @@
{#snippet viewContent(kind: string)}
{#if kind === "model"}
<div class="flex flex-col gap-3">
- <ModelSelector models={store.models} selected={store.activeModel} onSelect={handleSelectModel} />
+ <ModelSelector
+ models={store.models}
+ selected={store.activeModel}
+ onSelect={handleSelectModel}
+ modelInfo={store.modelInfo}
+ />
<!-- Keyed on the workspace conversation (active tab OR draft) so the inputs
re-mount per conversation — incl. switching between drafts — and can't
bleed across tabs. Editable for a draft too (cwd + effort apply from turn 1). -->
diff --git a/src/app/store.svelte.ts b/src/app/store.svelte.ts
index 78f6ede..41f3a5d 100644
--- a/src/app/store.svelte.ts
+++ b/src/app/store.svelte.ts
@@ -35,7 +35,7 @@ import type {
WarmResponse,
} from "@dispatch/transport-contract";
import type { SubscribeMessage, SurfaceServerMessage, SurfaceSpec } from "@dispatch/ui-contract";
-import type { ComputerEntry, ConversationStatus } from "@dispatch/wire";
+import type { ComputerEntry, ConversationStatus, ImageInput } from "@dispatch/wire";
import { untrack } from "svelte";
import { createIdbChunkStore } from "../adapters/idb";
import { createLocalStore } from "../adapters/local-storage";
@@ -160,7 +160,14 @@ export interface AppStore {
readonly storage: Storage | undefined;
/** The current spec for one surface by id (discovery-by-id), or null if absent. */
surface(surfaceId: string): SurfaceSpec | null;
- send(text: string): void;
+ /**
+ * Send a user message (start a turn). Forwards any staged `images`
+ * (`ImageInput[]` — base64 data URLs / https URLs) on the `chat.send` op;
+ * the server passes them to a vision-capable model natively or transcribes
+ * them via vision handoff for a non-vision model. Omitted on the wire when
+ * none are staged. On a draft, promotes to a tab first.
+ */
+ send(text: string, images?: readonly ImageInput[]): void;
/**
* Enqueue a steering message onto the focused conversation's queue
* (`chat.queue` WS op). While a turn is generating, the message is delivered
@@ -1149,7 +1156,7 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore {
return getSurfaceSpec(protocol, surfaceId);
},
- send(text: string): void {
+ send(text: string, images?: readonly ImageInput[]): void {
if (tabsStore.activeConversationId === null) {
// Draft: promote to tab on first send
const conversationId = draftConversationId;
@@ -1177,9 +1184,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore {
void refreshReasoningEffort();
void refreshCompactPercent();
// Now send on the promoted store
- chatStores.get(conversationId)?.send(text);
+ chatStores.get(conversationId)?.send(text, images);
} else {
- activeChat.send(text);
+ activeChat.send(text, images);
}
},
diff --git a/src/app/store.test.ts b/src/app/store.test.ts
index 947a9b0..7130c4d 100644
--- a/src/app/store.test.ts
+++ b/src/app/store.test.ts
@@ -423,6 +423,34 @@ describe("createAppStore", () => {
store.dispose();
});
+ it("sending from draft forwards staged images on chat.send", () => {
+ const ws = fakeSocket();
+ const store = createAppStore({
+ socketFactory: () => ws,
+ fetchImpl: fakeFetchImpl(),
+ localStorage: createFakeStorage(),
+ });
+ ws.resolveOpen();
+ ws.sent.length = 0;
+
+ const images = [
+ { url: "data:image/png;base64,AAAA", mimeType: "image/png" },
+ { url: "https://example.com/x.jpg" },
+ ];
+ store.send("describe these", images);
+
+ const msgs = parseSent(ws);
+ const chatSend = msgs.find((m) => (m as { type: string }).type === "chat.send") as
+ | { type: string; message: string; images?: { url: string }[] }
+ | undefined;
+ expect(chatSend).toBeTruthy();
+ expect(chatSend?.images).toEqual(images);
+ // The optimistic echo includes the image chunks.
+ expect(store.activeChat.chunks.some((c) => c.chunk.type === "image")).toBe(true);
+
+ store.dispose();
+ });
+
it("an incoming chat.delta renders in the transcript", () => {
const ws = fakeSocket();
const store = createAppStore({