summaryrefslogtreecommitdiffhomepage
path: root/src/features/chat/ui.test.ts
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-27 04:18:59 +0900
committerAdam Malczewski <[email protected]>2026-06-27 04:18:59 +0900
commit3566a20ebbded754070fce66af48690d1a904879 (patch)
treead4941abdbe685e2d4ae85d9ad3d0b6e9cf82c2d /src/features/chat/ui.test.ts
parent2fa03f8d7410c2b8d6be8e10ad088863e83d7177 (diff)
downloaddispatch-web-3566a20ebbded754070fce66af48690d1a904879.tar.gz
dispatch-web-3566a20ebbded754070fce66af48690d1a904879.zip
feat(vision): image paste + transcript image rendering + vision badge
Vision & vision-handoff frontend (consumes the backend's additive [email protected] / [email protected] image types — no version bump). Contracts mirrored: - .dispatch/wire.reference.md: ImageChunk added to the Chunk union + ImageChunk/ImageInput interfaces. - .dispatch/transport-contract.reference.md: ChatRequest.images, ModelMetadata.vision, + ImageChunk/ImageInput re-exports. Core (core/chunks): - conformance: assertChunkExhaustive handles the new 'image' variant (the guard caught it — its purpose). - appendUserMessage(state, text, images?) echoes a [text, image, ...] user run; the user-message event dedup scans the trailing user run (not just the last chunk) so an image-bearing echo doesn't duplicate the text; applyHistory's during-gen dedup matches a multi-chunk echo by content equality (chunkContentEquals + trailingRun helpers). UI: - ChatView renders user 'image' chunks as lazy <img> bubbles; a non-vision model's persisted [image, analysis-text] both render. read_image tool renders generically (no special-casing). - Composer: clipboard paste / file picker / drag-drop of images -> base64 data URLs, thumbnail previews with remove, forwarded on chat.send (omitted when none). Image-only sends allowed; steering (chat.queue) never forwards images. - ModelSelector: vision badge (isVisionModel) marks vision-capable models; indicator shows native-vision vs vision-handoff hint. Store wiring: ChatStore.send + AppStore.send + App.svelte handleSend thread images through; chat.send still omits cwd (only images added). Verification: svelte-check 0/0; vitest 901/901 (run twice, +34 new); biome clean; vite build OK. See backend-handoff.md §2j. Not merged or pushed.
Diffstat (limited to 'src/features/chat/ui.test.ts')
-rw-r--r--src/features/chat/ui.test.ts272
1 files changed, 269 insertions, 3 deletions
diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts
index a2fd944..af32f72 100644
--- a/src/features/chat/ui.test.ts
+++ b/src/features/chat/ui.test.ts
@@ -607,6 +607,87 @@ describe("ChatView", () => {
// Turn total should NOT render (total is null — turn still in progress)
expect(screen.queryByText(/^turn/)).toBeNull();
});
+
+ it("renders a user image chunk as an <img> with the chunk's url", () => {
+ const url = "data:image/png;base64,AAAA";
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: { type: "image", url, mimeType: "image/png" },
+ provisional: false,
+ },
+ ];
+
+ const { container } = render(ChatView, { props: { chunks } });
+
+ const img = container.querySelector("img");
+ expect(img).not.toBeNull();
+ expect(img?.getAttribute("src")).toBe(url);
+ expect(img?.getAttribute("alt")).toBe("image/png");
+ expect(img?.getAttribute("loading")).toBe("lazy");
+ });
+
+ it("renders a multi-chunk user message [text, image] and a transcription text", () => {
+ // A non-vision model: the server persists the original image chunk AND a
+ // transcription text chunk in the SAME user message — render both.
+ const url = "data:image/png;base64,BBQ=";
+ const chunks: RenderedChunk[] = [
+ { seq: 1, role: "user", chunk: { type: "text", text: "describe this" }, provisional: false },
+ {
+ seq: 2,
+ role: "user",
+ chunk: { type: "image", url, mimeType: "image/png" },
+ provisional: false,
+ },
+ {
+ seq: 3,
+ role: "user",
+ chunk: { type: "text", text: "[Image analysis (via kimi/k2)]: a red square" },
+ provisional: false,
+ },
+ ];
+
+ const { container } = render(ChatView, { props: { chunks } });
+
+ expect(screen.getByText("describe this")).toBeInTheDocument();
+ expect(screen.getByText(/\[Image analysis/)).toBeInTheDocument();
+ expect(container.querySelector("img")?.getAttribute("src")).toBe(url);
+ });
+
+ it("renders a read_image tool call/result like any other tool", () => {
+ // The read_image tool is available to all models; it is a normal tool call.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "assistant",
+ chunk: {
+ type: "tool-call",
+ toolCallId: "tc1",
+ toolName: "read_image",
+ input: { path: "/tmp/screenshot.png" },
+ },
+ provisional: false,
+ },
+ {
+ seq: 2,
+ role: "tool",
+ chunk: {
+ type: "tool-result",
+ toolCallId: "tc1",
+ toolName: "read_image",
+ content: "a screenshot of the desktop",
+ isError: false,
+ },
+ provisional: false,
+ },
+ ];
+
+ render(ChatView, { props: { chunks } });
+
+ expect(screen.getAllByText("read_image").length).toBeGreaterThan(0);
+ expect(screen.getByText("a screenshot of the desktop")).toBeInTheDocument();
+ });
});
describe("Composer", () => {
@@ -623,7 +704,7 @@ describe("Composer", () => {
await user.click(sendButton);
expect(onSend).toHaveBeenCalledTimes(1);
- expect(onSend).toHaveBeenCalledWith("Hello world");
+ expect(onSend).toHaveBeenCalledWith("Hello world", undefined);
expect(textarea).toHaveValue("");
});
@@ -651,7 +732,7 @@ describe("Composer", () => {
const sendButton = screen.getByRole("button", { name: "Send" });
await user.click(sendButton);
- expect(onSend).toHaveBeenCalledWith("hello");
+ expect(onSend).toHaveBeenCalledWith("hello", undefined);
});
it("sends on Enter key (without Shift)", async () => {
@@ -663,7 +744,7 @@ describe("Composer", () => {
const textarea = screen.getByRole("textbox", { name: "Message input" });
await user.type(textarea, "Test message{Enter}");
- expect(onSend).toHaveBeenCalledWith("Test message");
+ expect(onSend).toHaveBeenCalledWith("Test message", undefined);
});
it("does not send on Shift+Enter", async () => {
@@ -677,6 +758,142 @@ describe("Composer", () => {
expect(onSend).not.toHaveBeenCalled();
});
+
+ it("stages a pasted image and forwards it on send", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "look at this");
+
+ // jsdom has no ClipboardEvent/DataTransfer: dispatch a plain paste event
+ // carrying a mock clipboardData whose only item is an image File.
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const paste = new Event("paste", { bubbles: true });
+ Object.defineProperty(paste, "clipboardData", {
+ value: {
+ items: [{ kind: "file", type: "image/png", getAsFile: () => file }],
+ },
+ });
+ container.querySelector("textarea")?.dispatchEvent(paste);
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ await user.click(screen.getByRole("button", { name: "Send" }));
+
+ expect(onSend).toHaveBeenCalledTimes(1);
+ const [, images] = onSend.mock.calls[0] ?? [];
+ expect(images).toHaveLength(1);
+ expect(images[0]?.url).toMatch(/^data:image\/png;base64,/);
+ expect(images[0]?.mimeType).toBe("image/png");
+ });
+
+ it("lets a text paste proceed when no image is on the clipboard", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "hello");
+
+ // A text-only paste: no file items → the component must NOT preventDefault,
+ // so the default text paste path is unaffected (no image staged).
+ const paste = new Event("paste", { bubbles: true });
+ Object.defineProperty(paste, "clipboardData", {
+ value: { items: [{ kind: "string", type: "text/plain" }] },
+ });
+ container.querySelector("textarea")?.dispatchEvent(paste);
+
+ await new Promise((r) => setTimeout(r, 0));
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ });
+
+ it("stages an image via the attach button's file picker", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["JPG"], "photo.jpg", { type: "image/jpeg" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ // Image-only send (no text): the Send button is enabled.
+ const send = screen.getByRole("button", { name: "Send" });
+ expect(send).not.toBeDisabled();
+ await user.click(send);
+
+ expect(onSend).toHaveBeenCalledTimes(1);
+ const [text, images] = onSend.mock.calls[0] ?? [];
+ expect(text).toBe("");
+ expect(images).toHaveLength(1);
+ expect(images[0]?.mimeType).toBe("image/jpeg");
+ });
+
+ it("removes a staged image via the remove button", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+ await user.click(screen.getByRole("button", { name: "Remove image" }));
+
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ // With no text and no images, Send is disabled again.
+ expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
+ });
+
+ it("ignores a non-image file chosen via the picker", async () => {
+ const onSend = vi.fn();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["TXT"], "notes.txt", { type: "text/plain" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ // Give the async staging a chance; a non-image is skipped.
+ await new Promise((r) => setTimeout(r, 0));
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ });
+
+ it("queues (steers) text-only and never forwards images", async () => {
+ // While running, the Send button becomes "Queue"; steering is text-only.
+ const onQueue = vi.fn();
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend, onQueue, status: "running" } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "steer here");
+
+ // Also stage an image — it must NOT be forwarded on a queue.
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ await user.click(screen.getByRole("button", { name: "Queue" }));
+ expect(onQueue).toHaveBeenCalledWith("steer here");
+ expect(onSend).not.toHaveBeenCalled();
+ });
});
describe("ModelSelector", () => {
@@ -730,6 +947,55 @@ describe("ModelSelector", () => {
expect(onSelect).toHaveBeenCalledTimes(1);
expect(onSelect).toHaveBeenCalledWith("openai/gpt-4o");
});
+
+ it("marks vision-capable models in the model dropdown", () => {
+ const models = ["kimi/k2", "kimi/k1.5"];
+ const modelInfo = {
+ "kimi/k2": { vision: true },
+ "kimi/k1.5": { vision: false },
+ };
+ render(ModelSelector, {
+ props: { models, selected: "kimi/k2", onSelect: vi.fn(), modelInfo },
+ });
+
+ const modelSelect = screen.getByRole("combobox", { name: "Model selector" });
+ const options = within(modelSelect).getAllByRole("option");
+ expect(options).toHaveLength(2);
+ expect(options[0]?.textContent).toContain("vision");
+ expect(options[1]?.textContent).not.toContain("vision");
+ });
+
+ it("shows the vision indicator when the selected model is vision-capable", () => {
+ render(ModelSelector, {
+ props: {
+ models: ["kimi/k2"],
+ selected: "kimi/k2",
+ onSelect: vi.fn(),
+ modelInfo: { "kimi/k2": { vision: true } },
+ },
+ });
+ expect(screen.getByText(/sees images natively/)).toBeInTheDocument();
+ });
+
+ it("shows the vision-handoff hint when the selected model is non-vision", () => {
+ render(ModelSelector, {
+ props: {
+ models: ["umans/glm-5.2"],
+ selected: "umans/glm-5.2",
+ onSelect: vi.fn(),
+ modelInfo: { "umans/glm-5.2": { vision: false } },
+ },
+ });
+ expect(screen.getByText(/auto-described/i)).toBeInTheDocument();
+ expect(screen.queryByText(/sees images natively/)).not.toBeInTheDocument();
+ });
+
+ it("shows the handoff hint when modelInfo is absent", () => {
+ render(ModelSelector, {
+ props: { models: ["openai/gpt-4"], selected: "openai/gpt-4", onSelect: vi.fn() },
+ });
+ expect(screen.getByText(/auto-described/i)).toBeInTheDocument();
+ });
});
describe("ReasoningEffortSelector", () => {