diff options
| author | Adam Malczewski <[email protected]> | 2026-06-27 20:49:43 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-27 20:49:43 +0900 |
| commit | a59200e786f7d97d7ba5b9cd2bee9ffef531dac2 (patch) | |
| tree | 92088287487e34a9fee37f89325961110b1d3442 /src/features/chat | |
| parent | a9ca756de8cd023c0f2cb9954f344fff11146bc2 (diff) | |
| parent | b70ae547fdcb8c1794981957485537dc21a8b5fd (diff) | |
| download | dispatch-web-a59200e786f7d97d7ba5b9cd2bee9ffef531dac2.tar.gz dispatch-web-a59200e786f7d97d7ba5b9cd2bee9ffef531dac2.zip | |
Merge branch 'feature/vision-handoff' into dev
# Conflicts:
# .dispatch/transport-contract.reference.md
# backend-handoff.md
# src/app/App.svelte
# src/features/chat/ui/Composer.svelte
Diffstat (limited to 'src/features/chat')
| -rw-r--r-- | src/features/chat/index.ts | 3 | ||||
| -rw-r--r-- | src/features/chat/model-select.test.ts | 33 | ||||
| -rw-r--r-- | src/features/chat/model-select.ts | 15 | ||||
| -rw-r--r-- | src/features/chat/store.svelte.ts | 16 | ||||
| -rw-r--r-- | src/features/chat/store.test.ts | 89 | ||||
| -rw-r--r-- | src/features/chat/ui.test.ts | 365 | ||||
| -rw-r--r-- | src/features/chat/ui/ChatView.svelte | 25 | ||||
| -rw-r--r-- | src/features/chat/ui/Composer.svelte | 229 | ||||
| -rw-r--r-- | src/features/chat/ui/ModelSelector.svelte | 46 |
9 files changed, 789 insertions, 32 deletions
diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts index 5419b89..cf57cea 100644 --- a/src/features/chat/index.ts +++ b/src/features/chat/index.ts @@ -4,8 +4,9 @@ export type { RenderGroup, ToolBatchEntry, } from "../../core/chunks"; -export { groupRenderedChunks, viewProviderRetry } from "../../core/chunks"; +export { groupRenderedChunks, resolveImageUrl, viewProviderRetry } from "../../core/chunks"; export type { TurnMetricsEntry } from "../../core/metrics"; +export { isVisionModel } from "./model-select"; export type { ChatTransport, HistorySync, HistoryWindow, MetricsSync } from "./ports"; export type { EffortOption, diff --git a/src/features/chat/model-select.test.ts b/src/features/chat/model-select.test.ts index deb673d..6d3081d 100644 --- a/src/features/chat/model-select.test.ts +++ b/src/features/chat/model-select.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; -import { joinModelName, modelKeys, modelsForKey, splitModelName } from "./model-select"; +import { + isVisionModel, + joinModelName, + modelKeys, + modelsForKey, + splitModelName, +} from "./model-select"; describe("splitModelName", () => { it("splits on the first slash", () => { @@ -56,3 +62,28 @@ describe("modelsForKey", () => { expect(modelsForKey(["openai/gpt-4"], "anthropic")).toEqual([]); }); }); + +describe("isVisionModel", () => { + it("returns true when modelInfo[name].vision is true", () => { + const info = { "kimi/k2": { vision: true } }; + expect(isVisionModel(info, "kimi/k2")).toBe(true); + }); + + it("returns false when vision is false", () => { + const info = { "umans/glm-5.2": { vision: false } }; + expect(isVisionModel(info, "umans/glm-5.2")).toBe(false); + }); + + it("returns false when vision is absent (unknown)", () => { + const info = { "umans/glm-5.2": { contextWindow: 128000 } }; + expect(isVisionModel(info, "umans/glm-5.2")).toBe(false); + }); + + it("returns false for a model with no metadata entry at all", () => { + expect(isVisionModel({}, "unknown/model")).toBe(false); + }); + + it("returns false for an empty modelInfo map", () => { + expect(isVisionModel({}, "kimi/k2")).toBe(false); + }); +}); diff --git a/src/features/chat/model-select.ts b/src/features/chat/model-select.ts index db41772..602e0ef 100644 --- a/src/features/chat/model-select.ts +++ b/src/features/chat/model-select.ts @@ -1,3 +1,5 @@ +import type { ModelMetadata } from "@dispatch/transport-contract"; + /** * Pure helpers for the two-step model picker. * @@ -47,3 +49,16 @@ export function modelsForKey(models: readonly string[], key: string): string[] { } return out; } + +/** + * Whether a given full model name (`<key>/<model>`) is vision-capable — i.e. + * `GET /models` `modelInfo[name].vision === true`. Absent/`false`/unknown → + * `false` (the server's vision handoff transcribes images to text for it). + * Pure lookup against the catalog metadata; zero DOM. + */ +export function isVisionModel( + modelInfo: Readonly<Record<string, ModelMetadata>>, + fullName: string, +): boolean { + return modelInfo[fullName]?.vision === true; +} diff --git a/src/features/chat/store.svelte.ts b/src/features/chat/store.svelte.ts index 3588da1..5278737 100644 --- a/src/features/chat/store.svelte.ts +++ b/src/features/chat/store.svelte.ts @@ -4,7 +4,7 @@ import type { ChatQueueMessage, ChatSendMessage, } from "@dispatch/transport-contract"; -import type { ChatMessage, StoredChunk, TurnProviderRetryEvent } from "@dispatch/wire"; +import type { ChatMessage, ImageInput, StoredChunk, TurnProviderRetryEvent } from "@dispatch/wire"; import type { RenderedChunk, TranscriptState } from "../../core/chunks"; import { appendUserMessage, @@ -109,7 +109,14 @@ export interface ChatStore { */ readonly thinkingKeyBase: number; handleDelta(msg: ChatDeltaMessage | ChatErrorMessage): void; - send(text: string): void; + /** + * Send a user message (start a turn via `chat.send`). Optimistically echoes + * the text + any `images` as provisional user chunks (`[text, image, …]` in + * order), then forwards them on the WS `chat.send` op. `images` is omitted on + * the wire when none are staged (text-only, backward compatible). An + * images-only send (empty text) is allowed — the message text is `""`. + */ + send(text: string, images?: readonly ImageInput[]): void; /** * Enqueue a steering message onto the conversation's queue (`chat.queue` * WS op). While a turn is generating, the message is delivered mid-turn at @@ -312,8 +319,8 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { } }, - send(text: string): void { - transcript = appendUserMessage(transcript, text); + send(text: string, images?: readonly ImageInput[]): void { + transcript = appendUserMessage(transcript, text, images); maybeTrim(); const msg: ChatSendMessage = { type: "chat.send", @@ -321,6 +328,7 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { message: text, ...(_model !== undefined ? { model: _model } : {}), ...(deps.workspaceId !== undefined ? { workspaceId: deps.workspaceId } : {}), + ...(images !== undefined && images.length > 0 ? { images: [...images] } : {}), }; deps.transport.send(msg); }, diff --git a/src/features/chat/store.test.ts b/src/features/chat/store.test.ts index c052d1b..8f36994 100644 --- a/src/features/chat/store.test.ts +++ b/src/features/chat/store.test.ts @@ -1,4 +1,4 @@ -import type { AgentEvent, StepId, StoredChunk } from "@dispatch/wire"; +import type { AgentEvent, ImageInput, StepId, StoredChunk } from "@dispatch/wire"; import { describe, expect, it, vi } from "vitest"; import { createChatStore } from "./store.svelte"; import { @@ -144,6 +144,93 @@ describe("createChatStore", () => { store.dispose(); }); + it("send forwards staged images on chat.send and echoes them provisionally", () => { + const transport = createFakeTransport(); + const historySync = createFakeHistorySync(); + const metricsSync = createFakeMetricsSync(); + const cache = createFakeCache(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: historySync.impl, + metricsSync: metricsSync.impl, + cache: cache.impl, + }); + + const images: ImageInput[] = [ + { url: "data:image/png;base64,AAAA", mimeType: "image/png" }, + { url: "https://example.com/cat.jpg" }, + ]; + store.send("look at this", images); + + expect(transport.sent).toHaveLength(1); + const msg = transport.sent[0]; + expect(msg?.type).toBe("chat.send"); + expect(msg?.message).toBe("look at this"); + expect(msg?.images).toEqual(images); + + // Optimistic echo: a text chunk + two image chunks, provisional. + const chunks = store.chunks; + expect(chunks).toHaveLength(3); + expect(chunks[0]?.chunk).toEqual({ type: "text", text: "look at this" }); + expect(chunks[1]?.chunk).toEqual({ type: "image", url: images[0]?.url, mimeType: "image/png" }); + expect(chunks[2]?.chunk).toEqual({ type: "image", url: images[1]?.url }); + + store.dispose(); + }); + + it("send omits images on the wire when none are staged (backward compatible)", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("just text"); + + expect(transport.sent[0]).not.toHaveProperty("images"); + store.dispose(); + }); + + it("send omits images on the wire for an empty array", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("just text", []); + + expect(transport.sent[0]).not.toHaveProperty("images"); + store.dispose(); + }); + + it("send allows an images-only message (empty text)", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("", [{ url: "data:image/png;base64,AAAA", mimeType: "image/png" }]); + + expect(transport.sent[0]?.message).toBe(""); + expect(transport.sent[0]?.images).toHaveLength(1); + // The echo is image-only (no text chunk). + expect(store.chunks).toHaveLength(1); + expect(store.chunks[0]?.chunk.type).toBe("image"); + store.dispose(); + }); + describe("queueMessage (chat.queue — steering)", () => { it("posts a chat.queue with conversationId + text", () => { const transport = createFakeTransport(); diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index a2fd944..5f8067d 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -607,6 +607,180 @@ describe("ChatView", () => { // Turn total should NOT render (total is null — turn still in progress) expect(screen.queryByText(/^turn/)).toBeNull(); }); + + it("renders a user image chunk as an <img> with the chunk's url", () => { + const url = "data:image/png;base64,AAAA"; + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url, mimeType: "image/png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { props: { chunks } }); + + const img = container.querySelector("img"); + expect(img).not.toBeNull(); + expect(img?.getAttribute("src")).toBe(url); + expect(img?.getAttribute("alt")).toBe("image/png"); + expect(img?.getAttribute("loading")).toBe("lazy"); + }); + + it("resolves a persisted image chunk's relative url against apiBaseUrl", () => { + // Persisted image chunks now carry a compact relative path (`/images/…`) + // served by the backend — prepend the API base to render them. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url: "/images/conv-123/abc-456.png", mimeType: "image/png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe( + "http://localhost:24203/images/conv-123/abc-456.png", + ); + }); + + it("passes a data URL through unchanged even with apiBaseUrl set (optimistic echo)", () => { + // The optimistic echo (what the FE just sent) is still a data URL; it must + // NOT be mangled by the base-URL prepend. + const dataUrl = "data:image/png;base64,iVBOR="; + const chunks: RenderedChunk[] = [ + { seq: null, role: "user", chunk: { type: "image", url: dataUrl }, provisional: true }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe(dataUrl); + }); + + it("leaves a relative image url root-relative when apiBaseUrl is absent", () => { + // No apiBaseUrl → a browser resolves `/images/…` against the document origin. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url: "/images/conv-1/x.png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { props: { chunks } }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe("/images/conv-1/x.png"); + }); + + it("renders a multi-chunk user message [text, image] and a transcription text", () => { + // A non-vision model: the server persists the original image chunk AND a + // transcription text chunk in the SAME user message — render both. + const url = "data:image/png;base64,BBQ="; + const chunks: RenderedChunk[] = [ + { seq: 1, role: "user", chunk: { type: "text", text: "describe this" }, provisional: false }, + { + seq: 2, + role: "user", + chunk: { type: "image", url, mimeType: "image/png" }, + provisional: false, + }, + { + seq: 3, + role: "user", + chunk: { type: "text", text: "[Image analysis (via kimi/k2)]: a red square" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { props: { chunks } }); + + expect(screen.getByText("describe this")).toBeInTheDocument(); + expect(screen.getByText(/\[Image analysis/)).toBeInTheDocument(); + expect(container.querySelector("img")?.getAttribute("src")).toBe(url); + }); + + it("renders a consult_vision tool call/result like any other tool", () => { + // read_image is GONE — replaced by consult_vision (opens a vision-model + // conversation, attaches the image + question, returns the answer). It is + // a normal tool call: rendered generically by toolName. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "assistant", + chunk: { + type: "tool-call", + toolCallId: "tc1", + toolName: "consult_vision", + input: { question: "what is in this image?", imageIds: [1] }, + }, + provisional: false, + }, + { + seq: 2, + role: "tool", + chunk: { + type: "tool-result", + toolCallId: "tc1", + toolName: "consult_vision", + content: "a red square on a white background", + isError: false, + }, + provisional: false, + }, + ]; + + render(ChatView, { props: { chunks } }); + + expect(screen.getAllByText("consult_vision").length).toBeGreaterThan(0); + expect(screen.getByText("a red square on a white background")).toBeInTheDocument(); + }); + + it("renders a non-vision placeholder text chunk as-is", () => { + // A non-vision model gets a numbered placeholder (a regular text chunk) + // instead of an auto-transcription. Renders like any text chunk. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { + type: "text", + text: "[Image 1 attached — call consult_vision with imageIds=[1] and a specific question to analyze it]", + }, + provisional: false, + }, + ]; + + render(ChatView, { props: { chunks } }); + + expect(screen.getByText(/\[Image 1 attached/)).toBeInTheDocument(); + expect(screen.getByText(/consult_vision with imageIds/)).toBeInTheDocument(); + }); + + it("renders a compacted-image text chunk as-is", () => { + // Image compaction transcribes old images to [Compacted image]: <desc>. + // Regular text chunk — render as-is. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "text", text: "[Compacted image]: a chart showing rising sales" }, + provisional: false, + }, + ]; + + render(ChatView, { props: { chunks } }); + + expect(screen.getByText(/\[Compacted image\]/)).toBeInTheDocument(); + expect(screen.getByText(/rising sales/)).toBeInTheDocument(); + }); }); describe("Composer", () => { @@ -623,7 +797,7 @@ describe("Composer", () => { await user.click(sendButton); expect(onSend).toHaveBeenCalledTimes(1); - expect(onSend).toHaveBeenCalledWith("Hello world"); + expect(onSend).toHaveBeenCalledWith("Hello world", undefined); expect(textarea).toHaveValue(""); }); @@ -651,7 +825,7 @@ describe("Composer", () => { const sendButton = screen.getByRole("button", { name: "Send" }); await user.click(sendButton); - expect(onSend).toHaveBeenCalledWith("hello"); + expect(onSend).toHaveBeenCalledWith("hello", undefined); }); it("sends on Enter key (without Shift)", async () => { @@ -663,7 +837,7 @@ describe("Composer", () => { const textarea = screen.getByRole("textbox", { name: "Message input" }); await user.type(textarea, "Test message{Enter}"); - expect(onSend).toHaveBeenCalledWith("Test message"); + expect(onSend).toHaveBeenCalledWith("Test message", undefined); }); it("does not send on Shift+Enter", async () => { @@ -677,6 +851,142 @@ describe("Composer", () => { expect(onSend).not.toHaveBeenCalled(); }); + + it("stages a pasted image and forwards it on send", async () => { + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend } }); + + const textarea = screen.getByRole("textbox", { name: "Message input" }); + await user.type(textarea, "look at this"); + + // jsdom has no ClipboardEvent/DataTransfer: dispatch a plain paste event + // carrying a mock clipboardData whose only item is an image File. + const file = new File(["PNG"], "shot.png", { type: "image/png" }); + const paste = new Event("paste", { bubbles: true }); + Object.defineProperty(paste, "clipboardData", { + value: { + items: [{ kind: "file", type: "image/png", getAsFile: () => file }], + }, + }); + container.querySelector("textarea")?.dispatchEvent(paste); + + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: "Send" })); + + expect(onSend).toHaveBeenCalledTimes(1); + const [, images] = onSend.mock.calls[0] ?? []; + expect(images).toHaveLength(1); + expect(images[0]?.url).toMatch(/^data:image\/png;base64,/); + expect(images[0]?.mimeType).toBe("image/png"); + }); + + it("lets a text paste proceed when no image is on the clipboard", async () => { + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend } }); + + const textarea = screen.getByRole("textbox", { name: "Message input" }); + await user.type(textarea, "hello"); + + // A text-only paste: no file items → the component must NOT preventDefault, + // so the default text paste path is unaffected (no image staged). + const paste = new Event("paste", { bubbles: true }); + Object.defineProperty(paste, "clipboardData", { + value: { items: [{ kind: "string", type: "text/plain" }] }, + }); + container.querySelector("textarea")?.dispatchEvent(paste); + + await new Promise((r) => setTimeout(r, 0)); + expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument(); + }); + + it("stages an image via the attach button's file picker", async () => { + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend } }); + + const file = new File(["JPG"], "photo.jpg", { type: "image/jpeg" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + + // Image-only send (no text): the Send button is enabled. + const send = screen.getByRole("button", { name: "Send" }); + expect(send).not.toBeDisabled(); + await user.click(send); + + expect(onSend).toHaveBeenCalledTimes(1); + const [text, images] = onSend.mock.calls[0] ?? []; + expect(text).toBe(""); + expect(images).toHaveLength(1); + expect(images[0]?.mimeType).toBe("image/jpeg"); + }); + + it("removes a staged image via the remove button", async () => { + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend } }); + + const file = new File(["PNG"], "shot.png", { type: "image/png" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: "Remove image" })); + + expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument(); + // With no text and no images, Send is disabled again. + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); + }); + + it("ignores a non-image file chosen via the picker", async () => { + const onSend = vi.fn(); + const { container } = render(Composer, { props: { onSend } }); + + const file = new File(["TXT"], "notes.txt", { type: "text/plain" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + + // Give the async staging a chance; a non-image is skipped. + await new Promise((r) => setTimeout(r, 0)); + expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument(); + }); + + it("queues (steers) text-only and never forwards images", async () => { + // While running, the Send button becomes "Queue"; steering is text-only. + const onQueue = vi.fn(); + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend, onQueue, status: "running" } }); + + const textarea = screen.getByRole("textbox", { name: "Message input" }); + await user.type(textarea, "steer here"); + + // Also stage an image — it must NOT be forwarded on a queue. + const file = new File(["PNG"], "shot.png", { type: "image/png" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: "Queue" })); + expect(onQueue).toHaveBeenCalledWith("steer here"); + expect(onSend).not.toHaveBeenCalled(); + }); }); describe("ModelSelector", () => { @@ -730,6 +1040,55 @@ describe("ModelSelector", () => { expect(onSelect).toHaveBeenCalledTimes(1); expect(onSelect).toHaveBeenCalledWith("openai/gpt-4o"); }); + + it("marks vision-capable models in the model dropdown", () => { + const models = ["kimi/k2", "kimi/k1.5"]; + const modelInfo = { + "kimi/k2": { vision: true }, + "kimi/k1.5": { vision: false }, + }; + render(ModelSelector, { + props: { models, selected: "kimi/k2", onSelect: vi.fn(), modelInfo }, + }); + + const modelSelect = screen.getByRole("combobox", { name: "Model selector" }); + const options = within(modelSelect).getAllByRole("option"); + expect(options).toHaveLength(2); + expect(options[0]?.textContent).toContain("vision"); + expect(options[1]?.textContent).not.toContain("vision"); + }); + + it("shows the vision indicator when the selected model is vision-capable", () => { + render(ModelSelector, { + props: { + models: ["kimi/k2"], + selected: "kimi/k2", + onSelect: vi.fn(), + modelInfo: { "kimi/k2": { vision: true } }, + }, + }); + expect(screen.getByText(/sees images natively/)).toBeInTheDocument(); + }); + + it("shows the vision-handoff hint when the selected model is non-vision", () => { + render(ModelSelector, { + props: { + models: ["umans/glm-5.2"], + selected: "umans/glm-5.2", + onSelect: vi.fn(), + modelInfo: { "umans/glm-5.2": { vision: false } }, + }, + }); + expect(screen.getByText(/auto-described/i)).toBeInTheDocument(); + expect(screen.queryByText(/sees images natively/)).not.toBeInTheDocument(); + }); + + it("shows the handoff hint when modelInfo is absent", () => { + render(ModelSelector, { + props: { models: ["openai/gpt-4"], selected: "openai/gpt-4", onSelect: vi.fn() }, + }); + expect(screen.getByText(/auto-described/i)).toBeInTheDocument(); + }); }); describe("ReasoningEffortSelector", () => { diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte index b49f7b4..e67ca5b 100644 --- a/src/features/chat/ui/ChatView.svelte +++ b/src/features/chat/ui/ChatView.svelte @@ -1,6 +1,6 @@ <script lang="ts"> import type { TurnProviderRetryEvent } from "@dispatch/wire"; - import { groupRenderedChunks, type RenderedChunk, viewProviderRetry } from "../index"; + import { groupRenderedChunks, resolveImageUrl, type RenderedChunk, viewProviderRetry } from "../index"; import { interleaveTurnMetrics, viewCacheRate, @@ -24,6 +24,7 @@ onShowEarlier, thinkingKeyBase = 0, providerRetry = null, + apiBaseUrl = "", }: { chunks: readonly RenderedChunk[]; turnMetrics?: readonly TurnMetricsEntry[]; @@ -44,6 +45,14 @@ * newest attempt + delay, and is cleared when content resumes / turn ends. */ providerRetry?: TurnProviderRetryEvent | null; + /** + * The HTTP API base URL (e.g. `http://localhost:24203`). Persisted image + * chunks carry a compact relative path (`/images/<conv>/<uuid>.png`); this + * base is prepended to render them. The optimistic echo's data URL and any + * absolute URL pass through unchanged (see `resolveImageUrl`). Defaults to + * "" (root-relative — a browser resolves `/images/…` against its origin). + */ + apiBaseUrl?: string; } = $props(); // True while a show-earlier page-in is awaited (disables the button). @@ -95,11 +104,23 @@ {#snippet chunkRow(rendered: RenderedChunk)} {#if rendered.role === "user"} - <!-- User: a speech bubble, left-aligned --> + <!-- User: a speech bubble, left-aligned. A user message may be multi-chunk + ([text, image, image, …]); each chunk renders in its own bubble. A + persisted image chunk's url is a compact relative path (`/images/…`) + served by the backend — resolve it against the API base. The + optimistic echo's data URL (and any absolute URL) passes through. --> <div class="chat chat-start"> <div class="chat-bubble chat-bubble-primary"> {#if rendered.chunk.type === "text"} <p>{rendered.chunk.text}</p> + {:else if rendered.chunk.type === "image"} + <img + src={resolveImageUrl(rendered.chunk.url, apiBaseUrl)} + alt={rendered.chunk.mimeType ?? "pasted image"} + loading="lazy" + decoding="async" + class="max-h-80 max-w-full rounded" + /> {/if} </div> </div> diff --git a/src/features/chat/ui/Composer.svelte b/src/features/chat/ui/Composer.svelte index 0327f93..afe1e3c 100644 --- a/src/features/chat/ui/Composer.svelte +++ b/src/features/chat/ui/Composer.svelte @@ -1,8 +1,19 @@ <script lang="ts"> + import type { ImageInput } from "@dispatch/wire"; import { computeContextUsage, formatCompactTokens } from "../../../core/metrics"; const FALLBACK_CONTEXT_WINDOW = 1_000_000; const MAX_LINES = 7; + /** Accept only raster images (the provider image-content formats). */ + const IMAGE_ACCEPT = "image/png,image/jpeg,image/gif,image/webp"; + /** Reject images larger than this before base64-encoding (keeps payloads sane). */ + const MAX_IMAGE_BYTES = 8 * 1024 * 1024; + + /** A staged image awaiting send: a stable id + the `ImageInput` to forward. */ + interface StagedImage { + readonly id: string; + readonly input: ImageInput; + } let { onSend, @@ -12,12 +23,18 @@ contextWindow = undefined, status = "idle", }: { - onSend: (text: string) => void; + /** + * Send a message (start a turn via `chat.send`). Carries any staged images + * as `ImageInput[]` (base64 data URLs or https URLs); the store forwards + * them on the WS `chat.send` op / `POST /chat` body. `images` is omitted + * (not an empty array) when none are staged, so the wire stays text-only. + */ + onSend: (text: string, images?: ImageInput[]) => void; /** * Enqueue a steering message (`chat.queue`). When provided AND the status * is `running`, the send button becomes a "Queue" button that steers the - * in-flight turn instead of starting a new one. When absent, `onSend` is - * used regardless (tests / non-steering contexts). + * in-flight turn instead of starting a new one. Steering is text-only — + * it never carries images (a mid-turn injection has no image surface). */ onQueue?: (text: string) => void; /** Stop the in-flight generation (`POST /conversations/:id/stop`). */ @@ -39,22 +56,30 @@ export type ComposerStatus = "idle" | "running" | "queued" | "error"; let text = $state(""); + let images = $state<StagedImage[]>([]); let inputEl: HTMLTextAreaElement | undefined; + let fileInputEl: HTMLInputElement | undefined; + let dragOver = $state(false); const hasText = $derived(text.trim().length > 0); + const hasImages = $derived(images.length > 0); + const canSend = $derived(hasText || hasImages); const effectiveMax = $derived(contextWindow ?? FALLBACK_CONTEXT_WINDOW); const usage = $derived(computeContextUsage(contextSize, effectiveMax)); const hasUsage = $derived(contextSize !== undefined); // One button, three modes: // - idle → "Send" (starts a turn via chat.send) - // - running/queued + text → "Queue" (steers via chat.queue) + // - running/queued + text → "Queue" (steers via chat.queue — text only) // - running/queued + empty → "Stop" (aborts via POST /stop) // (`queued` behaves like `running` — the turn is in flight, just waiting for a // concurrency slot; the user can still steer or stop it.) + // Steering never carries images: when running with images staged but no text, + // the images stay staged (queue is text-only). Images-without-text while running + // is an unusual case that still sends (the server auto-starts/resolves). const inFlight = $derived(status === "running" || status === "queued"); const buttonMode = $derived.by<"send" | "queue" | "stop">(() => { - if (inFlight && !hasText && onStop !== undefined) return "stop"; + if (inFlight && !hasText && !hasImages && onStop !== undefined) return "stop"; if (inFlight && hasText && onQueue !== undefined) return "queue"; return "send"; }); @@ -63,7 +88,7 @@ ? "Queued for a slot…" : status === "running" ? "Steer the conversation..." - : "Type a message...", + : "Type a message, paste or drop an image…", ); // As the window fills, escalate color: calm → warning → danger. @@ -94,15 +119,97 @@ resize(); }); + /** Read a File into a base64 data URL (`data:image/…;base64,…`). */ + function fileToDataUrl(file: File): Promise<string> { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + if (typeof reader.result === "string") resolve(reader.result); + else reject(new Error("unreadable image")); + }; + reader.onerror = () => reject(reader.error ?? new Error("read failed")); + reader.readAsDataURL(file); + }); + } + + let imgSeq = 0; + /** Stage a File as an image (skip non-images / oversized). Returns whether staged. */ + async function stageFile(file: File): Promise<boolean> { + if (!file.type.startsWith("image/")) return false; + if (file.size > MAX_IMAGE_BYTES) return false; + const url = await fileToDataUrl(file); + // Prefer the file's declared MIME; fall back to the data URL's prefix. + const mimeType = file.type || undefined; + const id = `img-${Date.now()}-${imgSeq++}`; + images = [...images, { id, input: { url, ...(mimeType ? { mimeType } : {}) } }]; + return true; + } + + function removeImage(id: string): void { + images = images.filter((img) => img.id !== id); + } + + /** Handle a paste anywhere in the form: extract image items from the clipboard. */ + async function handlePaste(e: ClipboardEvent): Promise<void> { + const items = e.clipboardData?.items; + if (items === undefined) return; + let hadImage = false; + const staged: File[] = []; + for (const item of items) { + if (item.kind === "file" && item.type.startsWith("image/")) { + const file = item.getAsFile(); + if (file !== null) { + staged.push(file); + hadImage = true; + } + } + } + if (!hadImage) return; // let the default text paste proceed + e.preventDefault(); // suppress pasting the image as a filename string + for (const file of staged) { + await stageFile(file); + } + } + + /** File-picker <input type="file"> change. */ + async function handleFilePick(e: Event): Promise<void> { + const target = e.currentTarget as HTMLInputElement; + const files = target.files; + if (files === null) return; + for (const file of files) { + await stageFile(file); + } + target.value = ""; // reset so picking the same file again re-fires change + } + + /** Drop images onto the composer. */ + async function handleDrop(e: DragEvent): Promise<void> { + dragOver = false; + const files = e.dataTransfer?.files; + if (files === undefined || files.length === 0) return; + const hadImage = Array.from(files).some((f) => f.type.startsWith("image/")); + if (!hadImage) return; + e.preventDefault(); + for (const file of files) { + await stageFile(file); + } + } + function handleSubmit(): void { const trimmed = text.trim(); - if (trimmed.length === 0) return; + // Allow a send with images even when text is empty (an image-only turn). + if (trimmed.length === 0 && !hasImages) return; if (buttonMode === "queue") { + // Steering is text-only — never forward images. onQueue?.(trimmed); } else { - onSend(trimmed); + const toSend: ImageInput[] | undefined = hasImages + ? images.map((img) => img.input) + : undefined; + onSend(trimmed, toSend); } text = ""; + images = []; } function handleKeydown(e: KeyboardEvent): void { @@ -119,17 +226,68 @@ e.preventDefault(); handleSubmit(); }} + ondrop={handleDrop} + ondragover={(e) => { + if (e.dataTransfer?.types?.includes("Files")) { + e.preventDefault(); + dragOver = true; + } + }} + ondragleave={() => (dragOver = false)} > - <!-- Top bar: expanding textarea + single context-aware button --> + <!-- Top bar: expanding textarea + image-attach button + single context-aware button --> <div class="flex items-end gap-2 px-4 pt-3 pb-2"> - <textarea - bind:this={inputEl} - class="textarea textarea-bordered flex-1 resize-none leading-normal !min-h-0 h-auto" - bind:value={text} - onkeydown={handleKeydown} - {placeholder} - rows="1" - aria-label="Message input"></textarea> + <div + class="flex-1" + onpaste={handlePaste} + class:border-2={dragOver} + class:border-primary={dragOver} + class:border-dashed={dragOver} + class:rounded={dragOver} + > + <textarea + bind:this={inputEl} + class="textarea textarea-bordered w-full resize-none leading-normal !min-h-0 h-auto" + bind:value={text} + onkeydown={handleKeydown} + {placeholder} + rows="1" + aria-label="Message input"></textarea> + </div> + + <!-- Hidden file picker (images only; multiple). --> + <input + bind:this={fileInputEl} + type="file" + accept={IMAGE_ACCEPT} + multiple + class="hidden" + onchange={handleFilePick} + /> + <!-- Attach image button (opens the file picker). --> + <button + class="btn btn-ghost btn-square shrink-0" + type="button" + aria-label="Attach image" + title="Attach image" + onclick={() => fileInputEl?.click()} + > + <svg + xmlns="http://www.w3.org/2000/svg" + viewBox="0 0 24 24" + fill="none" + stroke="currentColor" + stroke-width="2" + stroke-linecap="round" + stroke-linejoin="round" + class="h-5 w-5" + > + <rect x="3" y="3" width="18" height="18" rx="2" ry="2"></rect> + <circle cx="8.5" cy="8.5" r="1.5"></circle> + <polyline points="21 15 16 10 5 21"></polyline> + </svg> + </button> + {#if buttonMode === "stop"} <button class="btn btn-error w-20 shrink-0" @@ -140,12 +298,47 @@ Stop </button> {:else} - <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!hasText}> + <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!canSend}> {buttonMode === "queue" ? "Queue" : "Send"} </button> {/if} </div> + <!-- Staged image thumbnails (previews) with remove buttons. --> + {#if hasImages} + <div class="flex flex-wrap gap-2 px-4 pb-1"> + {#each images as img (img.id)} + <div class="group relative h-20 w-20 shrink-0 overflow-hidden rounded border border-base-300"> + <img + src={img.input.url} + alt={img.input.mimeType ?? "staged image"} + class="h-full w-full object-cover" + /> + <button + class="btn btn-circle btn-xs absolute right-0 top-0 bg-base-100/80 hover:bg-error hover:text-error-content" + type="button" + aria-label="Remove image" + onclick={() => removeImage(img.id)} + > + <svg + xmlns="http://www.w3.org/2000/svg" + viewBox="0 0 24 24" + fill="none" + stroke="currentColor" + stroke-width="3" + stroke-linecap="round" + stroke-linejoin="round" + class="h-3 w-3" + > + <line x1="18" y1="6" x2="6" y2="18"></line> + <line x1="6" y1="6" x2="18" y2="18"></line> + </svg> + </button> + </div> + {/each} + </div> + {/if} + <!-- Bottom status bar: status icon · context-window fill · token count --> <div class="flex items-center gap-2 px-4 pb-2 text-xs text-base-content/50"> <span class="shrink-0"> diff --git a/src/features/chat/ui/ModelSelector.svelte b/src/features/chat/ui/ModelSelector.svelte index 03acb79..11b9feb 100644 --- a/src/features/chat/ui/ModelSelector.svelte +++ b/src/features/chat/ui/ModelSelector.svelte @@ -1,20 +1,32 @@ <script lang="ts"> - import { joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select"; + import type { ModelMetadata } from "@dispatch/transport-contract"; + import { isVisionModel, joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select"; let { models, selected, onSelect, + modelInfo = {}, }: { models: readonly string[]; selected: string; onSelect: (model: string) => void; + /** + * Per-model metadata from `GET /models` (`{ [name]: ModelMetadata }`). + * Used to show a "vision" badge next to models with `vision: true` (they + * natively accept images; others rely on the server's vision handoff). + * Optional — absent metadata → no badge (treated as non-vision). + */ + modelInfo?: Readonly<Record<string, ModelMetadata>>; } = $props(); const keys = $derived(modelKeys(models)); const current = $derived(splitModelName(selected)); const keyModels = $derived(modelsForKey(models, current.key)); + // Whether the currently-selected full model name is vision-capable. + const selectedVision = $derived(isVisionModel(modelInfo, selected)); + // Switching key jumps to the first model available under it. function selectKey(key: string): void { const first = modelsForKey(models, key)[0] ?? ""; @@ -24,6 +36,11 @@ function selectModel(model: string): void { onSelect(joinModelName(current.key, model)); } + + // The full `<key>/<model>` name for a model suffix under the current key. + function fullNameFor(modelSuffix: string): string { + return joinModelName(current.key, modelSuffix); + } </script> <div class="flex flex-col gap-2"> @@ -44,7 +61,32 @@ aria-label="Model selector" > {#each keyModels as model (model)} - <option value={model}>{model}</option> + <option value={model}> + {model}{#if isVisionModel(modelInfo, fullNameFor(model))} · vision{/if} + </option> {/each} </select> + {#if selectedVision} + <div class="flex items-center gap-1 text-xs text-base-content/60"> + <svg + xmlns="http://www.w3.org/2000/svg" + viewBox="0 0 24 24" + fill="none" + stroke="currentColor" + stroke-width="2" + stroke-linecap="round" + stroke-linejoin="round" + class="h-3.5 w-3.5" + aria-hidden="true" + > + <path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"></path> + <circle cx="12" cy="12" r="3"></circle> + </svg> + <span>Vision — this model sees images natively</span> + </div> + {:else} + <div class="text-xs text-base-content/40"> + Pasted images are auto-described (vision handoff) + </div> + {/if} </div> |
