summaryrefslogtreecommitdiffhomepage
path: root/src
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-27 04:18:59 +0900
committerAdam Malczewski <[email protected]>2026-06-27 04:18:59 +0900
commit3566a20ebbded754070fce66af48690d1a904879 (patch)
treead4941abdbe685e2d4ae85d9ad3d0b6e9cf82c2d /src
parent2fa03f8d7410c2b8d6be8e10ad088863e83d7177 (diff)
downloaddispatch-web-3566a20ebbded754070fce66af48690d1a904879.tar.gz
dispatch-web-3566a20ebbded754070fce66af48690d1a904879.zip
feat(vision): image paste + transcript image rendering + vision badge
Vision & vision-handoff frontend (consumes the backend's additive [email protected] / [email protected] image types — no version bump). Contracts mirrored: - .dispatch/wire.reference.md: ImageChunk added to the Chunk union + ImageChunk/ImageInput interfaces. - .dispatch/transport-contract.reference.md: ChatRequest.images, ModelMetadata.vision, + ImageChunk/ImageInput re-exports. Core (core/chunks): - conformance: assertChunkExhaustive handles the new 'image' variant (the guard caught it — its purpose). - appendUserMessage(state, text, images?) echoes a [text, image, ...] user run; the user-message event dedup scans the trailing user run (not just the last chunk) so an image-bearing echo doesn't duplicate the text; applyHistory's during-gen dedup matches a multi-chunk echo by content equality (chunkContentEquals + trailingRun helpers). UI: - ChatView renders user 'image' chunks as lazy <img> bubbles; a non-vision model's persisted [image, analysis-text] both render. read_image tool renders generically (no special-casing). - Composer: clipboard paste / file picker / drag-drop of images -> base64 data URLs, thumbnail previews with remove, forwarded on chat.send (omitted when none). Image-only sends allowed; steering (chat.queue) never forwards images. - ModelSelector: vision badge (isVisionModel) marks vision-capable models; indicator shows native-vision vs vision-handoff hint. Store wiring: ChatStore.send + AppStore.send + App.svelte handleSend thread images through; chat.send still omits cwd (only images added). Verification: svelte-check 0/0; vitest 901/901 (run twice, +34 new); biome clean; vite build OK. See backend-handoff.md §2j. Not merged or pushed.
Diffstat (limited to 'src')
-rw-r--r--src/app/App.svelte13
-rw-r--r--src/app/store.svelte.ts17
-rw-r--r--src/app/store.test.ts28
-rw-r--r--src/core/chunks/reducer.test.ts153
-rw-r--r--src/core/chunks/reducer.ts153
-rw-r--r--src/core/wire/conformance.test.ts40
-rw-r--r--src/core/wire/conformance.ts2
-rw-r--r--src/features/chat/model-select.test.ts33
-rw-r--r--src/features/chat/model-select.ts15
-rw-r--r--src/features/chat/store.svelte.ts16
-rw-r--r--src/features/chat/store.test.ts89
-rw-r--r--src/features/chat/ui.test.ts272
-rw-r--r--src/features/chat/ui/ChatView.svelte12
-rw-r--r--src/features/chat/ui/Composer.svelte232
-rw-r--r--src/features/chat/ui/ModelSelector.svelte46
15 files changed, 1053 insertions, 68 deletions
diff --git a/src/app/App.svelte b/src/app/App.svelte
index 09be947..f240a06 100644
--- a/src/app/App.svelte
+++ b/src/app/App.svelte
@@ -1,5 +1,5 @@
<script lang="ts">
- import type { ReasoningEffort } from "@dispatch/transport-contract";
+ import type { ImageInput, ReasoningEffort } from "@dispatch/transport-contract";
import type { InvokeMessage } from "@dispatch/ui-contract";
import { tick } from "svelte";
import Table from "../components/Table.svelte";
@@ -241,8 +241,8 @@
store.invoke(msg.surfaceId, msg.actionId, msg.payload);
}
- function handleSend(text: string) {
- store.send(text);
+ function handleSend(text: string, images?: readonly ImageInput[]): void {
+ store.send(text, images);
}
function handleQueue(text: string) {
@@ -587,7 +587,12 @@
{#snippet viewContent(kind: string)}
{#if kind === "model"}
<div class="flex flex-col gap-3">
- <ModelSelector models={store.models} selected={store.activeModel} onSelect={handleSelectModel} />
+ <ModelSelector
+ models={store.models}
+ selected={store.activeModel}
+ onSelect={handleSelectModel}
+ modelInfo={store.modelInfo}
+ />
<!-- Keyed on the workspace conversation (active tab OR draft) so the inputs
re-mount per conversation — incl. switching between drafts — and can't
bleed across tabs. Editable for a draft too (cwd + effort apply from turn 1). -->
diff --git a/src/app/store.svelte.ts b/src/app/store.svelte.ts
index 78f6ede..41f3a5d 100644
--- a/src/app/store.svelte.ts
+++ b/src/app/store.svelte.ts
@@ -35,7 +35,7 @@ import type {
WarmResponse,
} from "@dispatch/transport-contract";
import type { SubscribeMessage, SurfaceServerMessage, SurfaceSpec } from "@dispatch/ui-contract";
-import type { ComputerEntry, ConversationStatus } from "@dispatch/wire";
+import type { ComputerEntry, ConversationStatus, ImageInput } from "@dispatch/wire";
import { untrack } from "svelte";
import { createIdbChunkStore } from "../adapters/idb";
import { createLocalStore } from "../adapters/local-storage";
@@ -160,7 +160,14 @@ export interface AppStore {
readonly storage: Storage | undefined;
/** The current spec for one surface by id (discovery-by-id), or null if absent. */
surface(surfaceId: string): SurfaceSpec | null;
- send(text: string): void;
+ /**
+ * Send a user message (start a turn). Forwards any staged `images`
+ * (`ImageInput[]` — base64 data URLs / https URLs) on the `chat.send` op;
+ * the server passes them to a vision-capable model natively or transcribes
+ * them via vision handoff for a non-vision model. Omitted on the wire when
+ * none are staged. On a draft, promotes to a tab first.
+ */
+ send(text: string, images?: readonly ImageInput[]): void;
/**
* Enqueue a steering message onto the focused conversation's queue
* (`chat.queue` WS op). While a turn is generating, the message is delivered
@@ -1149,7 +1156,7 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore {
return getSurfaceSpec(protocol, surfaceId);
},
- send(text: string): void {
+ send(text: string, images?: readonly ImageInput[]): void {
if (tabsStore.activeConversationId === null) {
// Draft: promote to tab on first send
const conversationId = draftConversationId;
@@ -1177,9 +1184,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore {
void refreshReasoningEffort();
void refreshCompactPercent();
// Now send on the promoted store
- chatStores.get(conversationId)?.send(text);
+ chatStores.get(conversationId)?.send(text, images);
} else {
- activeChat.send(text);
+ activeChat.send(text, images);
}
},
diff --git a/src/app/store.test.ts b/src/app/store.test.ts
index 947a9b0..7130c4d 100644
--- a/src/app/store.test.ts
+++ b/src/app/store.test.ts
@@ -423,6 +423,34 @@ describe("createAppStore", () => {
store.dispose();
});
+ it("sending from draft forwards staged images on chat.send", () => {
+ const ws = fakeSocket();
+ const store = createAppStore({
+ socketFactory: () => ws,
+ fetchImpl: fakeFetchImpl(),
+ localStorage: createFakeStorage(),
+ });
+ ws.resolveOpen();
+ ws.sent.length = 0;
+
+ const images = [
+ { url: "data:image/png;base64,AAAA", mimeType: "image/png" },
+ { url: "https://example.com/x.jpg" },
+ ];
+ store.send("describe these", images);
+
+ const msgs = parseSent(ws);
+ const chatSend = msgs.find((m) => (m as { type: string }).type === "chat.send") as
+ | { type: string; message: string; images?: { url: string }[] }
+ | undefined;
+ expect(chatSend).toBeTruthy();
+ expect(chatSend?.images).toEqual(images);
+ // The optimistic echo includes the image chunks.
+ expect(store.activeChat.chunks.some((c) => c.chunk.type === "image")).toBe(true);
+
+ store.dispose();
+ });
+
it("an incoming chat.delta renders in the transcript", () => {
const ws = fakeSocket();
const store = createAppStore({
diff --git a/src/core/chunks/reducer.test.ts b/src/core/chunks/reducer.test.ts
index 8a2e1b7..058552e 100644
--- a/src/core/chunks/reducer.test.ts
+++ b/src/core/chunks/reducer.test.ts
@@ -1,4 +1,5 @@
import type {
+ ImageInput,
StepId,
StoredChunk,
TurnDoneEvent,
@@ -869,3 +870,155 @@ describe("appendUserMessage", () => {
expect(s.provisional[1]?.chunk).toEqual({ type: "text", text: "user msg" });
});
});
+
+const PNG = "data:image/png;base64,AAAA";
+const JPG = "data:image/jpeg;base64,BBBB";
+
+describe("appendUserMessage — images (vision handoff)", () => {
+ it("echoes text then image chunks in order", () => {
+ const images: ImageInput[] = [
+ { url: PNG, mimeType: "image/png" },
+ { url: JPG, mimeType: "image/jpeg" },
+ ];
+ let s = initialState();
+ s = appendUserMessage(s, "what's this?", images);
+ const chunks = selectChunks(s);
+ expect(chunks).toHaveLength(3);
+ expect(chunks[0]?.role).toBe("user");
+ expect(chunks[0]?.chunk).toEqual({ type: "text", text: "what's this?" });
+ expect(chunks[1]?.chunk).toEqual({ type: "image", url: PNG, mimeType: "image/png" });
+ expect(chunks[2]?.chunk).toEqual({ type: "image", url: JPG, mimeType: "image/jpeg" });
+ expect(chunks.every((c) => c.provisional && c.seq === null)).toBe(true);
+ });
+
+ it("omits mimeType when not provided", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "look", [{ url: PNG }]);
+ const img = selectChunks(s)[1]?.chunk;
+ expect(img).toEqual({ type: "image", url: PNG });
+ expect(img).not.toHaveProperty("mimeType");
+ });
+
+ it("echoes images-only when text is empty (no text chunk)", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "", [{ url: PNG, mimeType: "image/png" }]);
+ const chunks = selectChunks(s);
+ expect(chunks).toHaveLength(1);
+ expect(chunks[0]?.chunk.type).toBe("image");
+ });
+
+ it("echoes nothing for empty text + empty images", () => {
+ let s = initialState();
+ s = foldEvent(s, turnStart("t1"));
+ s = foldEvent(s, textDelta("t1", "partial"));
+ s = appendUserMessage(s, "", []);
+ // Defensive flush still happened; no user chunk added.
+ const users = selectChunks(s).filter((c) => c.role === "user");
+ expect(users).toHaveLength(0);
+ expect(s.accumulating).toBeNull();
+ });
+
+ it("skips images with an empty url", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "hi", [
+ { url: "", mimeType: "image/png" },
+ { url: PNG, mimeType: "image/png" },
+ ]);
+ const chunks = selectChunks(s);
+ expect(chunks).toHaveLength(2); // text + the one valid image
+ expect(chunks[1]?.chunk.type).toBe("image");
+ });
+
+ it("groups text + images into one user ChatMessage", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "see this", [{ url: PNG }]);
+ const msgs = selectMessages(s);
+ expect(msgs).toHaveLength(1);
+ expect(msgs[0]?.role).toBe("user");
+ expect(msgs[0]?.chunks).toHaveLength(2);
+ });
+});
+
+describe("foldEvent — user-message dedups against a text+image echo", () => {
+ const userMessage = (text: string): TurnInputEvent => ({
+ type: "user-message",
+ conversationId: "c1",
+ turnId: "t1",
+ text,
+ });
+
+ it("does not duplicate the text when images follow it in the echo", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "hi", [{ url: PNG, mimeType: "image/png" }]);
+ expect(selectChunks(s)).toHaveLength(2); // text + image
+ s = foldEvent(s, userMessage("hi"));
+ const users = selectChunks(s).filter((c) => c.role === "user");
+ expect(users).toHaveLength(2); // unchanged — no duplicate text
+ expect(users[0]?.chunk.type).toBe("text");
+ expect(users[1]?.chunk.type).toBe("image");
+ expect(s.generating).toBe(true);
+ });
+
+ it("still appends when the echoed text differs", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "first", [{ url: PNG }]);
+ s = foldEvent(s, userMessage("second"));
+ const users = selectChunks(s).filter((c) => c.role === "user");
+ expect(users.filter((c) => c.chunk.type === "text")).toHaveLength(2);
+ });
+});
+
+describe("applyHistory — multi-chunk image echo is superseded by committed", () => {
+ it("drops the provisional [text, image] echo when committed arrives during generation", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "hi", [{ url: PNG, mimeType: "image/png" }]);
+ s = foldEvent(s, turnStart("t1"));
+ expect(s.provisional).toHaveLength(2);
+
+ s = applyHistory(s, [
+ storedChunk(1, "user", { type: "text", text: "hi" }),
+ storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }),
+ ]);
+ expect(s.provisional).toEqual([]);
+ expect(s.committed).toHaveLength(2);
+ expect(s.committed[0]?.chunk).toEqual({ type: "text", text: "hi" });
+ expect(s.committed[1]?.chunk).toEqual({ type: "image", url: PNG, mimeType: "image/png" });
+ });
+
+ it("keeps the echo when committed is only a partial match (img not yet persisted)", () => {
+ let s = initialState();
+ s = appendUserMessage(s, "hi", [
+ { url: PNG, mimeType: "image/png" },
+ { url: JPG, mimeType: "image/jpeg" },
+ ]);
+ s = foldEvent(s, turnStart("t1"));
+ // Only the text + first image have been persisted so far.
+ s = applyHistory(s, [
+ storedChunk(1, "user", { type: "text", text: "hi" }),
+ storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }),
+ ]);
+ // Echo is [text, img1, img2]; committed run is [text, img1] — not a full
+ // match (echo is longer) → keep the echo (turn-seal will drop it wholesale).
+ expect(s.provisional.length).toBeGreaterThan(0);
+ });
+
+ it("renders committed image chunks from history (a non-vision transcription turn)", () => {
+ // The server persists the original image chunk AND the transcription text
+ // in the SAME user message: render both (image, then analysis text).
+ let s = initialState();
+ s = applyHistory(s, [
+ storedChunk(1, "user", { type: "text", text: "describe this" }),
+ storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }),
+ storedChunk(3, "user", {
+ type: "text",
+ text: "[Image analysis (via kimi/k2)]: a red square",
+ }),
+ storedChunk(4, "assistant", { type: "text", text: "the square is red" }),
+ ]);
+ const msgs = selectMessages(s);
+ expect(msgs).toHaveLength(2); // one user message (3 chunks) + one assistant
+ expect(msgs[0]?.role).toBe("user");
+ expect(msgs[0]?.chunks).toHaveLength(3);
+ expect(msgs[0]?.chunks[1]).toEqual({ type: "image", url: PNG, mimeType: "image/png" });
+ });
+});
diff --git a/src/core/chunks/reducer.ts b/src/core/chunks/reducer.ts
index 2152de3..64e41b9 100644
--- a/src/core/chunks/reducer.ts
+++ b/src/core/chunks/reducer.ts
@@ -1,4 +1,5 @@
-import type { AgentEvent, Chunk, StoredChunk } from "@dispatch/wire";
+import type { AgentEvent, Chunk, ImageInput, Role, StoredChunk } from "@dispatch/wire";
+import { assertChunkExhaustive } from "../wire/conformance";
import type { AccumulatingChunk, ProvisionalChunk, TranscriptState } from "./types";
/** The initial empty transcript state. */
@@ -43,6 +44,63 @@ function flushAccumulating(
}
/**
+ * Content equality for two chunks of the SAME role (used to de-dup an
+ * optimistic echo against the authoritative committed version). Compares the
+ * discriminating payload only — `stepId` (generation provenance) is ignored
+ * since it is absent on provisional echoes but present on committed tool chunks.
+ */
+function chunkContentEquals(a: Chunk, b: Chunk): boolean {
+ if (a.type !== b.type) return false;
+ switch (a.type) {
+ case "text": {
+ const o = b as Extract<Chunk, { type: "text" }>;
+ return a.text === o.text;
+ }
+ case "thinking": {
+ const o = b as Extract<Chunk, { type: "thinking" }>;
+ return a.text === o.text;
+ }
+ case "image": {
+ const o = b as Extract<Chunk, { type: "image" }>;
+ return a.url === o.url;
+ }
+ case "error": {
+ const o = b as Extract<Chunk, { type: "error" }>;
+ return a.message === o.message && a.code === o.code;
+ }
+ case "system": {
+ const o = b as Extract<Chunk, { type: "system" }>;
+ return a.text === o.text;
+ }
+ case "tool-call": {
+ const o = b as Extract<Chunk, { type: "tool-call" }>;
+ return a.toolCallId === o.toolCallId && a.toolName === o.toolName;
+ }
+ case "tool-result": {
+ const o = b as Extract<Chunk, { type: "tool-result" }>;
+ return a.toolCallId === o.toolCallId && a.toolName === o.toolName && a.isError === o.isError;
+ }
+ default:
+ return assertChunkExhaustive(a) === assertChunkExhaustive(b);
+ }
+}
+
+/**
+ * The trailing run of consecutive same-role provisional chunks at the end of
+ * `provisional` (the optimistic echo of one message). Returns the start/end
+ * indices `[start, end)` into `provisional` (empty if none).
+ */
+function trailingRun(
+ provisional: readonly ProvisionalChunk[],
+ role: Role,
+): readonly ProvisionalChunk[] {
+ const end = provisional.length;
+ let start = end;
+ while (start > 0 && provisional[start - 1]?.role === role) start--;
+ return provisional.slice(start, end);
+}
+
+/**
* Merge authoritative seq-keyed chunks into the committed history.
* Dedupes by seq (new wins), keeps seq-monotonic order, idempotent.
* When sealedTurnId is set, drops all provisional chunks (now superseded)
@@ -78,21 +136,30 @@ export function applyHistory(
// During generation: if new committed chunks arrived, the provisional
// array may contain duplicates — the optimistic echo from `appendUserMessage`
- // is now backed by a committed chunk (CR-6: user message persisted at turn
- // start). Remove provisional chunks that match the last committed chunk
- // (role + chunk content), keeping only the accumulating (streaming) chunk.
+ // is now backed by committed chunks (CR-6: user message persisted at turn
+ // start). A user message may be multi-chunk (`[text, image, image, …]`), so
+ // match the trailing provisional user-run against the trailing committed
+ // user-run by content equality and drop the whole echo when it is fully
+ // backed. Leaves the accumulating (streaming) chunk untouched.
if (addedNew && state.generating && state.provisional.length > 0) {
- const lastCommitted = committed[committed.length - 1];
- if (lastCommitted !== undefined) {
- const provisional = state.provisional.filter((p) => {
- if (p.role !== lastCommitted.role) return true;
- if (p.chunk.type !== lastCommitted.chunk.type) return true;
- if (p.chunk.type === "text" && lastCommitted.chunk.type === "text") {
- return p.chunk.text !== lastCommitted.chunk.text;
- }
- return true;
- });
- return { ...state, committed, provisional, accumulating: state.accumulating };
+ const provRun = trailingRun(state.provisional, "user");
+ if (provRun.length > 0) {
+ const commRun = trailingRun(committed, "user");
+ // Drop the provisional user echo iff every echoed chunk is content-equal
+ // to the corresponding committed chunk (the server persisted the same
+ // message). A partial match (echo longer than committed) keeps the echo
+ // — the not-yet-committed tail stays until the turn seals.
+ const fullyBacked =
+ commRun.length >= provRun.length &&
+ provRun.every((p, i) => {
+ const c = commRun[i];
+ return c !== undefined && chunkContentEquals(p.chunk, c.chunk);
+ });
+ if (fullyBacked) {
+ const dropStart = state.provisional.length - provRun.length;
+ const provisional = state.provisional.slice(0, dropStart);
+ return { ...state, committed, provisional, accumulating: state.accumulating };
+ }
}
}
@@ -138,16 +205,18 @@ function reduceEvent(state: TranscriptState, event: AgentEvent): TranscriptState
// The turn's USER prompt, surfaced on the event stream (backend CR-3) so a
// WATCHER/late-joiner renders it mid-turn instead of waiting for seal. The
// SENDER already echoed its own prompt optimistically (`appendUserMessage`),
- // so DE-DUP: skip if the trailing provisional chunk is already an identical
- // user text chunk. A pure watcher has no such echo → it appends and renders.
+ // so DE-DUP: skip if the trailing provisional USER run already contains an
+ // identical user text chunk. The echo may be multi-chunk (`[text, image,
+ // image, …]` — `appendUserMessage` appends the text first, then images), so
+ // we scan the whole trailing user run, not just the last chunk (which would
+ // be an image when images were pasted). A pure watcher has no such echo →
+ // it appends and renders. The `user-message` event carries ONLY text (never
+ // images — images arrive via history/loadSince); an images-only send (empty
+ // text) emits no `user-message` and is not de-duped here.
if (event.text.length === 0) return state;
- const last = state.provisional[state.provisional.length - 1];
- if (
- last !== undefined &&
- last.role === "user" &&
- last.chunk.type === "text" &&
- last.chunk.text === event.text
- ) {
+ const run = trailingRun(state.provisional, "user");
+ const alreadyEchoed = run.some((p) => p.chunk.type === "text" && p.chunk.text === event.text);
+ if (alreadyEchoed) {
return { ...state, generating: true };
}
const provisional = flushAccumulating(state.provisional, state.accumulating);
@@ -335,15 +404,43 @@ export function foldEvent(state: TranscriptState, event: AgentEvent): Transcript
/**
* Optimistically append a user message to the provisional list.
* Flushes any in-progress accumulating chunk first (defensively).
- * The provisional user chunk is superseded when applyHistory receives
+ * The provisional user chunks are superseded when applyHistory receives
* the authoritative committed chunks after a turn seals.
+ *
+ * When `images` are provided, they are appended AFTER the text chunk (in
+ * order) as `image` chunks — matching the server's persisted layout
+ * (`[text, image, image, …]` in one user message). A text chunk is only
+ * appended when `text` is non-empty (an images-only send echoes just the
+ * images). The `user-message` event carries only text (never images), so its
+ * de-dup scans the trailing user run for the echoed text rather than just the
+ * last chunk.
*/
-export function appendUserMessage(state: TranscriptState, text: string): TranscriptState {
+export function appendUserMessage(
+ state: TranscriptState,
+ text: string,
+ images?: readonly ImageInput[],
+): TranscriptState {
const provisional = flushAccumulating(state.provisional, state.accumulating);
- const userChunk: Chunk = { type: "text", text };
+ const userChunks: Chunk[] = [];
+ if (text.length > 0) userChunks.push({ type: "text", text });
+ if (images !== undefined) {
+ for (const img of images) {
+ if (img.url.length === 0) continue;
+ const chunk: Chunk =
+ img.mimeType !== undefined
+ ? { type: "image", url: img.url, mimeType: img.mimeType }
+ : { type: "image", url: img.url };
+ userChunks.push(chunk);
+ }
+ }
+ if (userChunks.length === 0) {
+ // Nothing to echo (empty text + no images) — leave state unchanged but
+ // still flush any accumulating chunk defensively.
+ return { ...state, provisional, accumulating: null };
+ }
return {
...state,
- provisional: [...provisional, { role: "user", chunk: userChunk }],
+ provisional: [...provisional, ...userChunks.map((chunk) => ({ role: "user", chunk }) as const)],
accumulating: null,
};
}
diff --git a/src/core/wire/conformance.test.ts b/src/core/wire/conformance.test.ts
index 880af07..0d955d5 100644
--- a/src/core/wire/conformance.test.ts
+++ b/src/core/wire/conformance.test.ts
@@ -128,9 +128,32 @@ describe("classifies every Chunk type", () => {
},
{ type: "error" as const, message: "e" },
{ type: "system" as const, text: "s" },
+ { type: "image" as const, url: "data:image/png;base64,AAAA", mimeType: "image/png" },
];
const labels = chunks.map(assertChunkExhaustive);
- expect(labels).toEqual(["text", "thinking", "tool-call", "tool-result", "error", "system"]);
+ expect(labels).toEqual([
+ "text",
+ "thinking",
+ "tool-call",
+ "tool-result",
+ "error",
+ "system",
+ "image",
+ ]);
+ });
+
+ it("covers all 7 Chunk variants", () => {
+ // Keeps the exhaustive guard honest: a new Chunk.type variant must be added
+ // both here and to `assertChunkExhaustive` or the `satisfies never` errors.
+ expect([
+ "text",
+ "thinking",
+ "tool-call",
+ "tool-result",
+ "error",
+ "system",
+ "image",
+ ] as const).toHaveLength(7);
});
});
@@ -222,6 +245,21 @@ describe("ChatSendMessage shape is constructible", () => {
expect(msg.model).toBe("default/gpt-4");
expect(msg.cwd).toBe("/tmp");
});
+
+ it("constructs a ChatSendMessage with pasted images", () => {
+ const msg: ChatSendMessage = {
+ type: "chat.send",
+ conversationId: "c1",
+ message: "what's in this image?",
+ images: [
+ { url: "data:image/png;base64,AAAA", mimeType: "image/png" },
+ { url: "https://example.com/cat.jpg" },
+ ],
+ };
+ expect(msg.images).toHaveLength(2);
+ expect(msg.images?.[0]?.mimeType).toBe("image/png");
+ expect(msg.images?.[1]?.mimeType).toBeUndefined();
+ });
});
describe("ConversationHistoryResponse shape is constructible", () => {
diff --git a/src/core/wire/conformance.ts b/src/core/wire/conformance.ts
index 412d80a..bfa67dc 100644
--- a/src/core/wire/conformance.ts
+++ b/src/core/wire/conformance.ts
@@ -60,6 +60,8 @@ export function assertChunkExhaustive(chunk: Chunk): string {
return "error";
case "system":
return "system";
+ case "image":
+ return "image";
default:
return chunk satisfies never;
}
diff --git a/src/features/chat/model-select.test.ts b/src/features/chat/model-select.test.ts
index deb673d..6d3081d 100644
--- a/src/features/chat/model-select.test.ts
+++ b/src/features/chat/model-select.test.ts
@@ -1,5 +1,11 @@
import { describe, expect, it } from "vitest";
-import { joinModelName, modelKeys, modelsForKey, splitModelName } from "./model-select";
+import {
+ isVisionModel,
+ joinModelName,
+ modelKeys,
+ modelsForKey,
+ splitModelName,
+} from "./model-select";
describe("splitModelName", () => {
it("splits on the first slash", () => {
@@ -56,3 +62,28 @@ describe("modelsForKey", () => {
expect(modelsForKey(["openai/gpt-4"], "anthropic")).toEqual([]);
});
});
+
+describe("isVisionModel", () => {
+ it("returns true when modelInfo[name].vision is true", () => {
+ const info = { "kimi/k2": { vision: true } };
+ expect(isVisionModel(info, "kimi/k2")).toBe(true);
+ });
+
+ it("returns false when vision is false", () => {
+ const info = { "umans/glm-5.2": { vision: false } };
+ expect(isVisionModel(info, "umans/glm-5.2")).toBe(false);
+ });
+
+ it("returns false when vision is absent (unknown)", () => {
+ const info = { "umans/glm-5.2": { contextWindow: 128000 } };
+ expect(isVisionModel(info, "umans/glm-5.2")).toBe(false);
+ });
+
+ it("returns false for a model with no metadata entry at all", () => {
+ expect(isVisionModel({}, "unknown/model")).toBe(false);
+ });
+
+ it("returns false for an empty modelInfo map", () => {
+ expect(isVisionModel({}, "kimi/k2")).toBe(false);
+ });
+});
diff --git a/src/features/chat/model-select.ts b/src/features/chat/model-select.ts
index db41772..602e0ef 100644
--- a/src/features/chat/model-select.ts
+++ b/src/features/chat/model-select.ts
@@ -1,3 +1,5 @@
+import type { ModelMetadata } from "@dispatch/transport-contract";
+
/**
* Pure helpers for the two-step model picker.
*
@@ -47,3 +49,16 @@ export function modelsForKey(models: readonly string[], key: string): string[] {
}
return out;
}
+
+/**
+ * Whether a given full model name (`<key>/<model>`) is vision-capable — i.e.
+ * `GET /models` `modelInfo[name].vision === true`. Absent/`false`/unknown →
+ * `false` (the server's vision handoff transcribes images to text for it).
+ * Pure lookup against the catalog metadata; zero DOM.
+ */
+export function isVisionModel(
+ modelInfo: Readonly<Record<string, ModelMetadata>>,
+ fullName: string,
+): boolean {
+ return modelInfo[fullName]?.vision === true;
+}
diff --git a/src/features/chat/store.svelte.ts b/src/features/chat/store.svelte.ts
index 3588da1..5278737 100644
--- a/src/features/chat/store.svelte.ts
+++ b/src/features/chat/store.svelte.ts
@@ -4,7 +4,7 @@ import type {
ChatQueueMessage,
ChatSendMessage,
} from "@dispatch/transport-contract";
-import type { ChatMessage, StoredChunk, TurnProviderRetryEvent } from "@dispatch/wire";
+import type { ChatMessage, ImageInput, StoredChunk, TurnProviderRetryEvent } from "@dispatch/wire";
import type { RenderedChunk, TranscriptState } from "../../core/chunks";
import {
appendUserMessage,
@@ -109,7 +109,14 @@ export interface ChatStore {
*/
readonly thinkingKeyBase: number;
handleDelta(msg: ChatDeltaMessage | ChatErrorMessage): void;
- send(text: string): void;
+ /**
+ * Send a user message (start a turn via `chat.send`). Optimistically echoes
+ * the text + any `images` as provisional user chunks (`[text, image, …]` in
+ * order), then forwards them on the WS `chat.send` op. `images` is omitted on
+ * the wire when none are staged (text-only, backward compatible). An
+ * images-only send (empty text) is allowed — the message text is `""`.
+ */
+ send(text: string, images?: readonly ImageInput[]): void;
/**
* Enqueue a steering message onto the conversation's queue (`chat.queue`
* WS op). While a turn is generating, the message is delivered mid-turn at
@@ -312,8 +319,8 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore {
}
},
- send(text: string): void {
- transcript = appendUserMessage(transcript, text);
+ send(text: string, images?: readonly ImageInput[]): void {
+ transcript = appendUserMessage(transcript, text, images);
maybeTrim();
const msg: ChatSendMessage = {
type: "chat.send",
@@ -321,6 +328,7 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore {
message: text,
...(_model !== undefined ? { model: _model } : {}),
...(deps.workspaceId !== undefined ? { workspaceId: deps.workspaceId } : {}),
+ ...(images !== undefined && images.length > 0 ? { images: [...images] } : {}),
};
deps.transport.send(msg);
},
diff --git a/src/features/chat/store.test.ts b/src/features/chat/store.test.ts
index c052d1b..8f36994 100644
--- a/src/features/chat/store.test.ts
+++ b/src/features/chat/store.test.ts
@@ -1,4 +1,4 @@
-import type { AgentEvent, StepId, StoredChunk } from "@dispatch/wire";
+import type { AgentEvent, ImageInput, StepId, StoredChunk } from "@dispatch/wire";
import { describe, expect, it, vi } from "vitest";
import { createChatStore } from "./store.svelte";
import {
@@ -144,6 +144,93 @@ describe("createChatStore", () => {
store.dispose();
});
+ it("send forwards staged images on chat.send and echoes them provisionally", () => {
+ const transport = createFakeTransport();
+ const historySync = createFakeHistorySync();
+ const metricsSync = createFakeMetricsSync();
+ const cache = createFakeCache();
+ const store = createChatStore({
+ conversationId: CONV_ID,
+ transport: transport.impl,
+ historySync: historySync.impl,
+ metricsSync: metricsSync.impl,
+ cache: cache.impl,
+ });
+
+ const images: ImageInput[] = [
+ { url: "data:image/png;base64,AAAA", mimeType: "image/png" },
+ { url: "https://example.com/cat.jpg" },
+ ];
+ store.send("look at this", images);
+
+ expect(transport.sent).toHaveLength(1);
+ const msg = transport.sent[0];
+ expect(msg?.type).toBe("chat.send");
+ expect(msg?.message).toBe("look at this");
+ expect(msg?.images).toEqual(images);
+
+ // Optimistic echo: a text chunk + two image chunks, provisional.
+ const chunks = store.chunks;
+ expect(chunks).toHaveLength(3);
+ expect(chunks[0]?.chunk).toEqual({ type: "text", text: "look at this" });
+ expect(chunks[1]?.chunk).toEqual({ type: "image", url: images[0]?.url, mimeType: "image/png" });
+ expect(chunks[2]?.chunk).toEqual({ type: "image", url: images[1]?.url });
+
+ store.dispose();
+ });
+
+ it("send omits images on the wire when none are staged (backward compatible)", () => {
+ const transport = createFakeTransport();
+ const store = createChatStore({
+ conversationId: CONV_ID,
+ transport: transport.impl,
+ historySync: createFakeHistorySync().impl,
+ metricsSync: createFakeMetricsSync().impl,
+ cache: createFakeCache().impl,
+ });
+
+ store.send("just text");
+
+ expect(transport.sent[0]).not.toHaveProperty("images");
+ store.dispose();
+ });
+
+ it("send omits images on the wire for an empty array", () => {
+ const transport = createFakeTransport();
+ const store = createChatStore({
+ conversationId: CONV_ID,
+ transport: transport.impl,
+ historySync: createFakeHistorySync().impl,
+ metricsSync: createFakeMetricsSync().impl,
+ cache: createFakeCache().impl,
+ });
+
+ store.send("just text", []);
+
+ expect(transport.sent[0]).not.toHaveProperty("images");
+ store.dispose();
+ });
+
+ it("send allows an images-only message (empty text)", () => {
+ const transport = createFakeTransport();
+ const store = createChatStore({
+ conversationId: CONV_ID,
+ transport: transport.impl,
+ historySync: createFakeHistorySync().impl,
+ metricsSync: createFakeMetricsSync().impl,
+ cache: createFakeCache().impl,
+ });
+
+ store.send("", [{ url: "data:image/png;base64,AAAA", mimeType: "image/png" }]);
+
+ expect(transport.sent[0]?.message).toBe("");
+ expect(transport.sent[0]?.images).toHaveLength(1);
+ // The echo is image-only (no text chunk).
+ expect(store.chunks).toHaveLength(1);
+ expect(store.chunks[0]?.chunk.type).toBe("image");
+ store.dispose();
+ });
+
describe("queueMessage (chat.queue — steering)", () => {
it("posts a chat.queue with conversationId + text", () => {
const transport = createFakeTransport();
diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts
index a2fd944..af32f72 100644
--- a/src/features/chat/ui.test.ts
+++ b/src/features/chat/ui.test.ts
@@ -607,6 +607,87 @@ describe("ChatView", () => {
// Turn total should NOT render (total is null — turn still in progress)
expect(screen.queryByText(/^turn/)).toBeNull();
});
+
+ it("renders a user image chunk as an <img> with the chunk's url", () => {
+ const url = "data:image/png;base64,AAAA";
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: { type: "image", url, mimeType: "image/png" },
+ provisional: false,
+ },
+ ];
+
+ const { container } = render(ChatView, { props: { chunks } });
+
+ const img = container.querySelector("img");
+ expect(img).not.toBeNull();
+ expect(img?.getAttribute("src")).toBe(url);
+ expect(img?.getAttribute("alt")).toBe("image/png");
+ expect(img?.getAttribute("loading")).toBe("lazy");
+ });
+
+ it("renders a multi-chunk user message [text, image] and a transcription text", () => {
+ // A non-vision model: the server persists the original image chunk AND a
+ // transcription text chunk in the SAME user message — render both.
+ const url = "data:image/png;base64,BBQ=";
+ const chunks: RenderedChunk[] = [
+ { seq: 1, role: "user", chunk: { type: "text", text: "describe this" }, provisional: false },
+ {
+ seq: 2,
+ role: "user",
+ chunk: { type: "image", url, mimeType: "image/png" },
+ provisional: false,
+ },
+ {
+ seq: 3,
+ role: "user",
+ chunk: { type: "text", text: "[Image analysis (via kimi/k2)]: a red square" },
+ provisional: false,
+ },
+ ];
+
+ const { container } = render(ChatView, { props: { chunks } });
+
+ expect(screen.getByText("describe this")).toBeInTheDocument();
+ expect(screen.getByText(/\[Image analysis/)).toBeInTheDocument();
+ expect(container.querySelector("img")?.getAttribute("src")).toBe(url);
+ });
+
+ it("renders a read_image tool call/result like any other tool", () => {
+ // The read_image tool is available to all models; it is a normal tool call.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "assistant",
+ chunk: {
+ type: "tool-call",
+ toolCallId: "tc1",
+ toolName: "read_image",
+ input: { path: "/tmp/screenshot.png" },
+ },
+ provisional: false,
+ },
+ {
+ seq: 2,
+ role: "tool",
+ chunk: {
+ type: "tool-result",
+ toolCallId: "tc1",
+ toolName: "read_image",
+ content: "a screenshot of the desktop",
+ isError: false,
+ },
+ provisional: false,
+ },
+ ];
+
+ render(ChatView, { props: { chunks } });
+
+ expect(screen.getAllByText("read_image").length).toBeGreaterThan(0);
+ expect(screen.getByText("a screenshot of the desktop")).toBeInTheDocument();
+ });
});
describe("Composer", () => {
@@ -623,7 +704,7 @@ describe("Composer", () => {
await user.click(sendButton);
expect(onSend).toHaveBeenCalledTimes(1);
- expect(onSend).toHaveBeenCalledWith("Hello world");
+ expect(onSend).toHaveBeenCalledWith("Hello world", undefined);
expect(textarea).toHaveValue("");
});
@@ -651,7 +732,7 @@ describe("Composer", () => {
const sendButton = screen.getByRole("button", { name: "Send" });
await user.click(sendButton);
- expect(onSend).toHaveBeenCalledWith("hello");
+ expect(onSend).toHaveBeenCalledWith("hello", undefined);
});
it("sends on Enter key (without Shift)", async () => {
@@ -663,7 +744,7 @@ describe("Composer", () => {
const textarea = screen.getByRole("textbox", { name: "Message input" });
await user.type(textarea, "Test message{Enter}");
- expect(onSend).toHaveBeenCalledWith("Test message");
+ expect(onSend).toHaveBeenCalledWith("Test message", undefined);
});
it("does not send on Shift+Enter", async () => {
@@ -677,6 +758,142 @@ describe("Composer", () => {
expect(onSend).not.toHaveBeenCalled();
});
+
+ it("stages a pasted image and forwards it on send", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "look at this");
+
+ // jsdom has no ClipboardEvent/DataTransfer: dispatch a plain paste event
+ // carrying a mock clipboardData whose only item is an image File.
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const paste = new Event("paste", { bubbles: true });
+ Object.defineProperty(paste, "clipboardData", {
+ value: {
+ items: [{ kind: "file", type: "image/png", getAsFile: () => file }],
+ },
+ });
+ container.querySelector("textarea")?.dispatchEvent(paste);
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ await user.click(screen.getByRole("button", { name: "Send" }));
+
+ expect(onSend).toHaveBeenCalledTimes(1);
+ const [, images] = onSend.mock.calls[0] ?? [];
+ expect(images).toHaveLength(1);
+ expect(images[0]?.url).toMatch(/^data:image\/png;base64,/);
+ expect(images[0]?.mimeType).toBe("image/png");
+ });
+
+ it("lets a text paste proceed when no image is on the clipboard", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "hello");
+
+ // A text-only paste: no file items → the component must NOT preventDefault,
+ // so the default text paste path is unaffected (no image staged).
+ const paste = new Event("paste", { bubbles: true });
+ Object.defineProperty(paste, "clipboardData", {
+ value: { items: [{ kind: "string", type: "text/plain" }] },
+ });
+ container.querySelector("textarea")?.dispatchEvent(paste);
+
+ await new Promise((r) => setTimeout(r, 0));
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ });
+
+ it("stages an image via the attach button's file picker", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["JPG"], "photo.jpg", { type: "image/jpeg" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ // Image-only send (no text): the Send button is enabled.
+ const send = screen.getByRole("button", { name: "Send" });
+ expect(send).not.toBeDisabled();
+ await user.click(send);
+
+ expect(onSend).toHaveBeenCalledTimes(1);
+ const [text, images] = onSend.mock.calls[0] ?? [];
+ expect(text).toBe("");
+ expect(images).toHaveLength(1);
+ expect(images[0]?.mimeType).toBe("image/jpeg");
+ });
+
+ it("removes a staged image via the remove button", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+ await user.click(screen.getByRole("button", { name: "Remove image" }));
+
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ // With no text and no images, Send is disabled again.
+ expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
+ });
+
+ it("ignores a non-image file chosen via the picker", async () => {
+ const onSend = vi.fn();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["TXT"], "notes.txt", { type: "text/plain" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ // Give the async staging a chance; a non-image is skipped.
+ await new Promise((r) => setTimeout(r, 0));
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ });
+
+ it("queues (steers) text-only and never forwards images", async () => {
+ // While running, the Send button becomes "Queue"; steering is text-only.
+ const onQueue = vi.fn();
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend, onQueue, status: "running" } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "steer here");
+
+ // Also stage an image — it must NOT be forwarded on a queue.
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ await user.click(screen.getByRole("button", { name: "Queue" }));
+ expect(onQueue).toHaveBeenCalledWith("steer here");
+ expect(onSend).not.toHaveBeenCalled();
+ });
});
describe("ModelSelector", () => {
@@ -730,6 +947,55 @@ describe("ModelSelector", () => {
expect(onSelect).toHaveBeenCalledTimes(1);
expect(onSelect).toHaveBeenCalledWith("openai/gpt-4o");
});
+
+ it("marks vision-capable models in the model dropdown", () => {
+ const models = ["kimi/k2", "kimi/k1.5"];
+ const modelInfo = {
+ "kimi/k2": { vision: true },
+ "kimi/k1.5": { vision: false },
+ };
+ render(ModelSelector, {
+ props: { models, selected: "kimi/k2", onSelect: vi.fn(), modelInfo },
+ });
+
+ const modelSelect = screen.getByRole("combobox", { name: "Model selector" });
+ const options = within(modelSelect).getAllByRole("option");
+ expect(options).toHaveLength(2);
+ expect(options[0]?.textContent).toContain("vision");
+ expect(options[1]?.textContent).not.toContain("vision");
+ });
+
+ it("shows the vision indicator when the selected model is vision-capable", () => {
+ render(ModelSelector, {
+ props: {
+ models: ["kimi/k2"],
+ selected: "kimi/k2",
+ onSelect: vi.fn(),
+ modelInfo: { "kimi/k2": { vision: true } },
+ },
+ });
+ expect(screen.getByText(/sees images natively/)).toBeInTheDocument();
+ });
+
+ it("shows the vision-handoff hint when the selected model is non-vision", () => {
+ render(ModelSelector, {
+ props: {
+ models: ["umans/glm-5.2"],
+ selected: "umans/glm-5.2",
+ onSelect: vi.fn(),
+ modelInfo: { "umans/glm-5.2": { vision: false } },
+ },
+ });
+ expect(screen.getByText(/auto-described/i)).toBeInTheDocument();
+ expect(screen.queryByText(/sees images natively/)).not.toBeInTheDocument();
+ });
+
+ it("shows the handoff hint when modelInfo is absent", () => {
+ render(ModelSelector, {
+ props: { models: ["openai/gpt-4"], selected: "openai/gpt-4", onSelect: vi.fn() },
+ });
+ expect(screen.getByText(/auto-described/i)).toBeInTheDocument();
+ });
});
describe("ReasoningEffortSelector", () => {
diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte
index e72f639..8081951 100644
--- a/src/features/chat/ui/ChatView.svelte
+++ b/src/features/chat/ui/ChatView.svelte
@@ -95,11 +95,21 @@
{#snippet chunkRow(rendered: RenderedChunk)}
{#if rendered.role === "user"}
- <!-- User: a speech bubble, left-aligned -->
+ <!-- User: a speech bubble, left-aligned. A user message may be multi-chunk
+ ([text, image, image, …]); each chunk renders in its own bubble (the
+ image's url is a base64 data URL or an https URL — render it directly). -->
<div class="chat chat-start">
<div class="chat-bubble chat-bubble-primary">
{#if rendered.chunk.type === "text"}
<p>{rendered.chunk.text}</p>
+ {:else if rendered.chunk.type === "image"}
+ <img
+ src={rendered.chunk.url}
+ alt={rendered.chunk.mimeType ?? "pasted image"}
+ loading="lazy"
+ decoding="async"
+ class="max-h-80 max-w-full rounded"
+ />
{/if}
</div>
</div>
diff --git a/src/features/chat/ui/Composer.svelte b/src/features/chat/ui/Composer.svelte
index 96b4b3a..7898448 100644
--- a/src/features/chat/ui/Composer.svelte
+++ b/src/features/chat/ui/Composer.svelte
@@ -1,8 +1,19 @@
<script lang="ts">
+ import type { ImageInput } from "@dispatch/wire";
import { computeContextUsage, formatCompactTokens } from "../../../core/metrics";
const FALLBACK_CONTEXT_WINDOW = 1_000_000;
const MAX_LINES = 7;
+ /** Accept only raster images (the provider image-content formats). */
+ const IMAGE_ACCEPT = "image/png,image/jpeg,image/gif,image/webp";
+ /** Reject images larger than this before base64-encoding (keeps payloads sane). */
+ const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
+
+ /** A staged image awaiting send: a stable id + the `ImageInput` to forward. */
+ interface StagedImage {
+ readonly id: string;
+ readonly input: ImageInput;
+ }
let {
onSend,
@@ -12,12 +23,18 @@
contextWindow = undefined,
status = "idle",
}: {
- onSend: (text: string) => void;
+ /**
+ * Send a message (start a turn via `chat.send`). Carries any staged images
+ * as `ImageInput[]` (base64 data URLs or https URLs); the store forwards
+ * them on the WS `chat.send` op / `POST /chat` body. `images` is omitted
+ * (not an empty array) when none are staged, so the wire stays text-only.
+ */
+ onSend: (text: string, images?: ImageInput[]) => void;
/**
* Enqueue a steering message (`chat.queue`). When provided AND the status
* is `running`, the send button becomes a "Queue" button that steers the
- * in-flight turn instead of starting a new one. When absent, `onSend` is
- * used regardless (tests / non-steering contexts).
+ * in-flight turn instead of starting a new one. Steering is text-only —
+ * it never carries images (a mid-turn injection has no image surface).
*/
onQueue?: (text: string) => void;
/** Stop the in-flight generation (`POST /conversations/:id/stop`). */
@@ -32,24 +49,35 @@
} = $props();
let text = $state("");
+ let images = $state<StagedImage[]>([]);
let inputEl: HTMLTextAreaElement | undefined;
+ let fileInputEl: HTMLInputElement | undefined;
+ let dragOver = $state(false);
const hasText = $derived(text.trim().length > 0);
+ const hasImages = $derived(images.length > 0);
+ const canSend = $derived(hasText || hasImages);
const effectiveMax = $derived(contextWindow ?? FALLBACK_CONTEXT_WINDOW);
const usage = $derived(computeContextUsage(contextSize, effectiveMax));
const hasUsage = $derived(contextSize !== undefined);
// One button, three modes:
// - idle → "Send" (starts a turn via chat.send)
- // - running + text → "Queue" (steers via chat.queue)
+ // - running + text/images → "Queue" (steers via chat.queue — text only)
// - running + empty → "Stop" (aborts via POST /stop)
+ // Steering never carries images: when running with images staged but no text,
+ // fall back to "Send" semantics is wrong mid-turn — instead queue the text part
+ // only (images stay staged). Simplest correct rule: queue is text-only and only
+ // offered with text; images-without-text while running is an unusual case that
+ // still sends (the server auto-starts/resolves). Keep the three-mode logic
+ // driven by text presence for the queue vs stop split.
const buttonMode = $derived.by<"send" | "queue" | "stop">(() => {
- if (status === "running" && !hasText && onStop !== undefined) return "stop";
+ if (status === "running" && !hasText && !hasImages && onStop !== undefined) return "stop";
if (status === "running" && hasText && onQueue !== undefined) return "queue";
return "send";
});
const placeholder = $derived(
- status === "running" ? "Steer the conversation..." : "Type a message...",
+ status === "running" ? "Steer the conversation..." : "Type a message, paste or drop an image…",
);
// As the window fills, escalate color: calm → warning → danger.
@@ -80,15 +108,97 @@
resize();
});
+ /** Read a File into a base64 data URL (`data:image/…;base64,…`). */
+ function fileToDataUrl(file: File): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => {
+ if (typeof reader.result === "string") resolve(reader.result);
+ else reject(new Error("unreadable image"));
+ };
+ reader.onerror = () => reject(reader.error ?? new Error("read failed"));
+ reader.readAsDataURL(file);
+ });
+ }
+
+ let imgSeq = 0;
+ /** Stage a File as an image (skip non-images / oversized). Returns whether staged. */
+ async function stageFile(file: File): Promise<boolean> {
+ if (!file.type.startsWith("image/")) return false;
+ if (file.size > MAX_IMAGE_BYTES) return false;
+ const url = await fileToDataUrl(file);
+ // Prefer the file's declared MIME; fall back to the data URL's prefix.
+ const mimeType = file.type || undefined;
+ const id = `img-${Date.now()}-${imgSeq++}`;
+ images = [...images, { id, input: { url, ...(mimeType ? { mimeType } : {}) } }];
+ return true;
+ }
+
+ function removeImage(id: string): void {
+ images = images.filter((img) => img.id !== id);
+ }
+
+ /** Handle a paste anywhere in the form: extract image items from the clipboard. */
+ async function handlePaste(e: ClipboardEvent): Promise<void> {
+ const items = e.clipboardData?.items;
+ if (items === undefined) return;
+ let hadImage = false;
+ const staged: File[] = [];
+ for (const item of items) {
+ if (item.kind === "file" && item.type.startsWith("image/")) {
+ const file = item.getAsFile();
+ if (file !== null) {
+ staged.push(file);
+ hadImage = true;
+ }
+ }
+ }
+ if (!hadImage) return; // let the default text paste proceed
+ e.preventDefault(); // suppress pasting the image as a filename string
+ for (const file of staged) {
+ await stageFile(file);
+ }
+ }
+
+ /** File-picker <input type="file"> change. */
+ async function handleFilePick(e: Event): Promise<void> {
+ const target = e.currentTarget as HTMLInputElement;
+ const files = target.files;
+ if (files === null) return;
+ for (const file of files) {
+ await stageFile(file);
+ }
+ target.value = ""; // reset so picking the same file again re-fires change
+ }
+
+ /** Drop images onto the composer. */
+ async function handleDrop(e: DragEvent): Promise<void> {
+ dragOver = false;
+ const files = e.dataTransfer?.files;
+ if (files === undefined || files.length === 0) return;
+ const hadImage = Array.from(files).some((f) => f.type.startsWith("image/"));
+ if (!hadImage) return;
+ e.preventDefault();
+ for (const file of files) {
+ await stageFile(file);
+ }
+ }
+
function handleSubmit(): void {
const trimmed = text.trim();
- if (trimmed.length === 0) return;
+ // Allow a send with images even when text is empty (an image-only turn).
+ if (trimmed.length === 0 && !hasImages) return;
if (buttonMode === "queue") {
+ // Steering is text-only — never forward images.
onQueue?.(trimmed);
} else {
- onSend(trimmed);
+ const toSend: ImageInput[] | undefined = hasImages
+ ? images.map((img) => img.input)
+ : undefined;
+ onSend(trimmed, toSend);
}
text = "";
+ images = [];
}
function handleKeydown(e: KeyboardEvent): void {
@@ -105,17 +215,68 @@
e.preventDefault();
handleSubmit();
}}
+ ondrop={handleDrop}
+ ondragover={(e) => {
+ if (e.dataTransfer?.types?.includes("Files")) {
+ e.preventDefault();
+ dragOver = true;
+ }
+ }}
+ ondragleave={() => (dragOver = false)}
>
- <!-- Top bar: expanding textarea + single context-aware button -->
+ <!-- Top bar: expanding textarea + image-attach button + single context-aware button -->
<div class="flex items-end gap-2 px-4 pt-3 pb-2">
- <textarea
- bind:this={inputEl}
- class="textarea textarea-bordered flex-1 resize-none leading-normal !min-h-0 h-auto"
- bind:value={text}
- onkeydown={handleKeydown}
- {placeholder}
- rows="1"
- aria-label="Message input"></textarea>
+ <div
+ class="flex-1"
+ onpaste={handlePaste}
+ class:border-2={dragOver}
+ class:border-primary={dragOver}
+ class:border-dashed={dragOver}
+ class:rounded={dragOver}
+ >
+ <textarea
+ bind:this={inputEl}
+ class="textarea textarea-bordered w-full resize-none leading-normal !min-h-0 h-auto"
+ bind:value={text}
+ onkeydown={handleKeydown}
+ {placeholder}
+ rows="1"
+ aria-label="Message input"></textarea>
+ </div>
+
+ <!-- Hidden file picker (images only; multiple). -->
+ <input
+ bind:this={fileInputEl}
+ type="file"
+ accept={IMAGE_ACCEPT}
+ multiple
+ class="hidden"
+ onchange={handleFilePick}
+ />
+ <!-- Attach image button (opens the file picker). -->
+ <button
+ class="btn btn-ghost btn-square shrink-0"
+ type="button"
+ aria-label="Attach image"
+ title="Attach image"
+ onclick={() => fileInputEl?.click()}
+ >
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ stroke-width="2"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ class="h-5 w-5"
+ >
+ <rect x="3" y="3" width="18" height="18" rx="2" ry="2"></rect>
+ <circle cx="8.5" cy="8.5" r="1.5"></circle>
+ <polyline points="21 15 16 10 5 21"></polyline>
+ </svg>
+ </button>
+
{#if buttonMode === "stop"}
<button
class="btn btn-error w-20 shrink-0"
@@ -126,12 +287,47 @@
Stop
</button>
{:else}
- <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!hasText}>
+ <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!canSend}>
{buttonMode === "queue" ? "Queue" : "Send"}
</button>
{/if}
</div>
+ <!-- Staged image thumbnails (previews) with remove buttons. -->
+ {#if hasImages}
+ <div class="flex flex-wrap gap-2 px-4 pb-1">
+ {#each images as img (img.id)}
+ <div class="group relative h-20 w-20 shrink-0 overflow-hidden rounded border border-base-300">
+ <img
+ src={img.input.url}
+ alt={img.input.mimeType ?? "staged image"}
+ class="h-full w-full object-cover"
+ />
+ <button
+ class="btn btn-circle btn-xs absolute right-0 top-0 bg-base-100/80 hover:bg-error hover:text-error-content"
+ type="button"
+ aria-label="Remove image"
+ onclick={() => removeImage(img.id)}
+ >
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ stroke-width="3"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ class="h-3 w-3"
+ >
+ <line x1="18" y1="6" x2="6" y2="18"></line>
+ <line x1="6" y1="6" x2="18" y2="18"></line>
+ </svg>
+ </button>
+ </div>
+ {/each}
+ </div>
+ {/if}
+
<!-- Bottom status bar: status icon · context-window fill · token count -->
<div class="flex items-center gap-2 px-4 pb-2 text-xs text-base-content/50">
<span class="shrink-0">
diff --git a/src/features/chat/ui/ModelSelector.svelte b/src/features/chat/ui/ModelSelector.svelte
index 03acb79..11b9feb 100644
--- a/src/features/chat/ui/ModelSelector.svelte
+++ b/src/features/chat/ui/ModelSelector.svelte
@@ -1,20 +1,32 @@
<script lang="ts">
- import { joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select";
+ import type { ModelMetadata } from "@dispatch/transport-contract";
+ import { isVisionModel, joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select";
let {
models,
selected,
onSelect,
+ modelInfo = {},
}: {
models: readonly string[];
selected: string;
onSelect: (model: string) => void;
+ /**
+ * Per-model metadata from `GET /models` (`{ [name]: ModelMetadata }`).
+ * Used to show a "vision" badge next to models with `vision: true` (they
+ * natively accept images; others rely on the server's vision handoff).
+ * Optional — absent metadata → no badge (treated as non-vision).
+ */
+ modelInfo?: Readonly<Record<string, ModelMetadata>>;
} = $props();
const keys = $derived(modelKeys(models));
const current = $derived(splitModelName(selected));
const keyModels = $derived(modelsForKey(models, current.key));
+ // Whether the currently-selected full model name is vision-capable.
+ const selectedVision = $derived(isVisionModel(modelInfo, selected));
+
// Switching key jumps to the first model available under it.
function selectKey(key: string): void {
const first = modelsForKey(models, key)[0] ?? "";
@@ -24,6 +36,11 @@
function selectModel(model: string): void {
onSelect(joinModelName(current.key, model));
}
+
+ // The full `<key>/<model>` name for a model suffix under the current key.
+ function fullNameFor(modelSuffix: string): string {
+ return joinModelName(current.key, modelSuffix);
+ }
</script>
<div class="flex flex-col gap-2">
@@ -44,7 +61,32 @@
aria-label="Model selector"
>
{#each keyModels as model (model)}
- <option value={model}>{model}</option>
+ <option value={model}>
+ {model}{#if isVisionModel(modelInfo, fullNameFor(model))} · vision{/if}
+ </option>
{/each}
</select>
+ {#if selectedVision}
+ <div class="flex items-center gap-1 text-xs text-base-content/60">
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ stroke-width="2"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ class="h-3.5 w-3.5"
+ aria-hidden="true"
+ >
+ <path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"></path>
+ <circle cx="12" cy="12" r="3"></circle>
+ </svg>
+ <span>Vision — this model sees images natively</span>
+ </div>
+ {:else}
+ <div class="text-xs text-base-content/40">
+ Pasted images are auto-described (vision handoff)
+ </div>
+ {/if}
</div>