summaryrefslogtreecommitdiffhomepage
path: root/src/features/chat
diff options
context:
space:
mode:
Diffstat (limited to 'src/features/chat')
-rw-r--r--src/features/chat/index.ts3
-rw-r--r--src/features/chat/model-select.test.ts33
-rw-r--r--src/features/chat/model-select.ts15
-rw-r--r--src/features/chat/store.svelte.ts16
-rw-r--r--src/features/chat/store.test.ts89
-rw-r--r--src/features/chat/ui.test.ts365
-rw-r--r--src/features/chat/ui/ChatView.svelte25
-rw-r--r--src/features/chat/ui/Composer.svelte229
-rw-r--r--src/features/chat/ui/ModelSelector.svelte46
9 files changed, 789 insertions, 32 deletions
diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts
index 5419b89..cf57cea 100644
--- a/src/features/chat/index.ts
+++ b/src/features/chat/index.ts
@@ -4,8 +4,9 @@ export type {
RenderGroup,
ToolBatchEntry,
} from "../../core/chunks";
-export { groupRenderedChunks, viewProviderRetry } from "../../core/chunks";
+export { groupRenderedChunks, resolveImageUrl, viewProviderRetry } from "../../core/chunks";
export type { TurnMetricsEntry } from "../../core/metrics";
+export { isVisionModel } from "./model-select";
export type { ChatTransport, HistorySync, HistoryWindow, MetricsSync } from "./ports";
export type {
EffortOption,
diff --git a/src/features/chat/model-select.test.ts b/src/features/chat/model-select.test.ts
index deb673d..6d3081d 100644
--- a/src/features/chat/model-select.test.ts
+++ b/src/features/chat/model-select.test.ts
@@ -1,5 +1,11 @@
import { describe, expect, it } from "vitest";
-import { joinModelName, modelKeys, modelsForKey, splitModelName } from "./model-select";
+import {
+ isVisionModel,
+ joinModelName,
+ modelKeys,
+ modelsForKey,
+ splitModelName,
+} from "./model-select";
describe("splitModelName", () => {
it("splits on the first slash", () => {
@@ -56,3 +62,28 @@ describe("modelsForKey", () => {
expect(modelsForKey(["openai/gpt-4"], "anthropic")).toEqual([]);
});
});
+
+describe("isVisionModel", () => {
+ it("returns true when modelInfo[name].vision is true", () => {
+ const info = { "kimi/k2": { vision: true } };
+ expect(isVisionModel(info, "kimi/k2")).toBe(true);
+ });
+
+ it("returns false when vision is false", () => {
+ const info = { "umans/glm-5.2": { vision: false } };
+ expect(isVisionModel(info, "umans/glm-5.2")).toBe(false);
+ });
+
+ it("returns false when vision is absent (unknown)", () => {
+ const info = { "umans/glm-5.2": { contextWindow: 128000 } };
+ expect(isVisionModel(info, "umans/glm-5.2")).toBe(false);
+ });
+
+ it("returns false for a model with no metadata entry at all", () => {
+ expect(isVisionModel({}, "unknown/model")).toBe(false);
+ });
+
+ it("returns false for an empty modelInfo map", () => {
+ expect(isVisionModel({}, "kimi/k2")).toBe(false);
+ });
+});
diff --git a/src/features/chat/model-select.ts b/src/features/chat/model-select.ts
index db41772..602e0ef 100644
--- a/src/features/chat/model-select.ts
+++ b/src/features/chat/model-select.ts
@@ -1,3 +1,5 @@
+import type { ModelMetadata } from "@dispatch/transport-contract";
+
/**
* Pure helpers for the two-step model picker.
*
@@ -47,3 +49,16 @@ export function modelsForKey(models: readonly string[], key: string): string[] {
}
return out;
}
+
+/**
+ * Whether a given full model name (`<key>/<model>`) is vision-capable — i.e.
+ * `GET /models` `modelInfo[name].vision === true`. Absent/`false`/unknown →
+ * `false` (the server's vision handoff transcribes images to text for it).
+ * Pure lookup against the catalog metadata; zero DOM.
+ */
+export function isVisionModel(
+ modelInfo: Readonly<Record<string, ModelMetadata>>,
+ fullName: string,
+): boolean {
+ return modelInfo[fullName]?.vision === true;
+}
diff --git a/src/features/chat/store.svelte.ts b/src/features/chat/store.svelte.ts
index 3588da1..5278737 100644
--- a/src/features/chat/store.svelte.ts
+++ b/src/features/chat/store.svelte.ts
@@ -4,7 +4,7 @@ import type {
ChatQueueMessage,
ChatSendMessage,
} from "@dispatch/transport-contract";
-import type { ChatMessage, StoredChunk, TurnProviderRetryEvent } from "@dispatch/wire";
+import type { ChatMessage, ImageInput, StoredChunk, TurnProviderRetryEvent } from "@dispatch/wire";
import type { RenderedChunk, TranscriptState } from "../../core/chunks";
import {
appendUserMessage,
@@ -109,7 +109,14 @@ export interface ChatStore {
*/
readonly thinkingKeyBase: number;
handleDelta(msg: ChatDeltaMessage | ChatErrorMessage): void;
- send(text: string): void;
+ /**
+ * Send a user message (start a turn via `chat.send`). Optimistically echoes
+ * the text + any `images` as provisional user chunks (`[text, image, …]` in
+ * order), then forwards them on the WS `chat.send` op. `images` is omitted on
+ * the wire when none are staged (text-only, backward compatible). An
+ * images-only send (empty text) is allowed — the message text is `""`.
+ */
+ send(text: string, images?: readonly ImageInput[]): void;
/**
* Enqueue a steering message onto the conversation's queue (`chat.queue`
* WS op). While a turn is generating, the message is delivered mid-turn at
@@ -312,8 +319,8 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore {
}
},
- send(text: string): void {
- transcript = appendUserMessage(transcript, text);
+ send(text: string, images?: readonly ImageInput[]): void {
+ transcript = appendUserMessage(transcript, text, images);
maybeTrim();
const msg: ChatSendMessage = {
type: "chat.send",
@@ -321,6 +328,7 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore {
message: text,
...(_model !== undefined ? { model: _model } : {}),
...(deps.workspaceId !== undefined ? { workspaceId: deps.workspaceId } : {}),
+ ...(images !== undefined && images.length > 0 ? { images: [...images] } : {}),
};
deps.transport.send(msg);
},
diff --git a/src/features/chat/store.test.ts b/src/features/chat/store.test.ts
index c052d1b..8f36994 100644
--- a/src/features/chat/store.test.ts
+++ b/src/features/chat/store.test.ts
@@ -1,4 +1,4 @@
-import type { AgentEvent, StepId, StoredChunk } from "@dispatch/wire";
+import type { AgentEvent, ImageInput, StepId, StoredChunk } from "@dispatch/wire";
import { describe, expect, it, vi } from "vitest";
import { createChatStore } from "./store.svelte";
import {
@@ -144,6 +144,93 @@ describe("createChatStore", () => {
store.dispose();
});
+ it("send forwards staged images on chat.send and echoes them provisionally", () => {
+ const transport = createFakeTransport();
+ const historySync = createFakeHistorySync();
+ const metricsSync = createFakeMetricsSync();
+ const cache = createFakeCache();
+ const store = createChatStore({
+ conversationId: CONV_ID,
+ transport: transport.impl,
+ historySync: historySync.impl,
+ metricsSync: metricsSync.impl,
+ cache: cache.impl,
+ });
+
+ const images: ImageInput[] = [
+ { url: "data:image/png;base64,AAAA", mimeType: "image/png" },
+ { url: "https://example.com/cat.jpg" },
+ ];
+ store.send("look at this", images);
+
+ expect(transport.sent).toHaveLength(1);
+ const msg = transport.sent[0];
+ expect(msg?.type).toBe("chat.send");
+ expect(msg?.message).toBe("look at this");
+ expect(msg?.images).toEqual(images);
+
+ // Optimistic echo: a text chunk + two image chunks, provisional.
+ const chunks = store.chunks;
+ expect(chunks).toHaveLength(3);
+ expect(chunks[0]?.chunk).toEqual({ type: "text", text: "look at this" });
+ expect(chunks[1]?.chunk).toEqual({ type: "image", url: images[0]?.url, mimeType: "image/png" });
+ expect(chunks[2]?.chunk).toEqual({ type: "image", url: images[1]?.url });
+
+ store.dispose();
+ });
+
+ it("send omits images on the wire when none are staged (backward compatible)", () => {
+ const transport = createFakeTransport();
+ const store = createChatStore({
+ conversationId: CONV_ID,
+ transport: transport.impl,
+ historySync: createFakeHistorySync().impl,
+ metricsSync: createFakeMetricsSync().impl,
+ cache: createFakeCache().impl,
+ });
+
+ store.send("just text");
+
+ expect(transport.sent[0]).not.toHaveProperty("images");
+ store.dispose();
+ });
+
+ it("send omits images on the wire for an empty array", () => {
+ const transport = createFakeTransport();
+ const store = createChatStore({
+ conversationId: CONV_ID,
+ transport: transport.impl,
+ historySync: createFakeHistorySync().impl,
+ metricsSync: createFakeMetricsSync().impl,
+ cache: createFakeCache().impl,
+ });
+
+ store.send("just text", []);
+
+ expect(transport.sent[0]).not.toHaveProperty("images");
+ store.dispose();
+ });
+
+ it("send allows an images-only message (empty text)", () => {
+ const transport = createFakeTransport();
+ const store = createChatStore({
+ conversationId: CONV_ID,
+ transport: transport.impl,
+ historySync: createFakeHistorySync().impl,
+ metricsSync: createFakeMetricsSync().impl,
+ cache: createFakeCache().impl,
+ });
+
+ store.send("", [{ url: "data:image/png;base64,AAAA", mimeType: "image/png" }]);
+
+ expect(transport.sent[0]?.message).toBe("");
+ expect(transport.sent[0]?.images).toHaveLength(1);
+ // The echo is image-only (no text chunk).
+ expect(store.chunks).toHaveLength(1);
+ expect(store.chunks[0]?.chunk.type).toBe("image");
+ store.dispose();
+ });
+
describe("queueMessage (chat.queue — steering)", () => {
it("posts a chat.queue with conversationId + text", () => {
const transport = createFakeTransport();
diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts
index a2fd944..5f8067d 100644
--- a/src/features/chat/ui.test.ts
+++ b/src/features/chat/ui.test.ts
@@ -607,6 +607,180 @@ describe("ChatView", () => {
// Turn total should NOT render (total is null — turn still in progress)
expect(screen.queryByText(/^turn/)).toBeNull();
});
+
+ it("renders a user image chunk as an <img> with the chunk's url", () => {
+ const url = "data:image/png;base64,AAAA";
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: { type: "image", url, mimeType: "image/png" },
+ provisional: false,
+ },
+ ];
+
+ const { container } = render(ChatView, { props: { chunks } });
+
+ const img = container.querySelector("img");
+ expect(img).not.toBeNull();
+ expect(img?.getAttribute("src")).toBe(url);
+ expect(img?.getAttribute("alt")).toBe("image/png");
+ expect(img?.getAttribute("loading")).toBe("lazy");
+ });
+
+ it("resolves a persisted image chunk's relative url against apiBaseUrl", () => {
+ // Persisted image chunks now carry a compact relative path (`/images/…`)
+ // served by the backend — prepend the API base to render them.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: { type: "image", url: "/images/conv-123/abc-456.png", mimeType: "image/png" },
+ provisional: false,
+ },
+ ];
+
+ const { container } = render(ChatView, {
+ props: { chunks, apiBaseUrl: "http://localhost:24203" },
+ });
+
+ expect(container.querySelector("img")?.getAttribute("src")).toBe(
+ "http://localhost:24203/images/conv-123/abc-456.png",
+ );
+ });
+
+ it("passes a data URL through unchanged even with apiBaseUrl set (optimistic echo)", () => {
+ // The optimistic echo (what the FE just sent) is still a data URL; it must
+ // NOT be mangled by the base-URL prepend.
+ const dataUrl = "data:image/png;base64,iVBOR=";
+ const chunks: RenderedChunk[] = [
+ { seq: null, role: "user", chunk: { type: "image", url: dataUrl }, provisional: true },
+ ];
+
+ const { container } = render(ChatView, {
+ props: { chunks, apiBaseUrl: "http://localhost:24203" },
+ });
+
+ expect(container.querySelector("img")?.getAttribute("src")).toBe(dataUrl);
+ });
+
+ it("leaves a relative image url root-relative when apiBaseUrl is absent", () => {
+ // No apiBaseUrl → a browser resolves `/images/…` against the document origin.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: { type: "image", url: "/images/conv-1/x.png" },
+ provisional: false,
+ },
+ ];
+
+ const { container } = render(ChatView, { props: { chunks } });
+
+ expect(container.querySelector("img")?.getAttribute("src")).toBe("/images/conv-1/x.png");
+ });
+
+ it("renders a multi-chunk user message [text, image] and a transcription text", () => {
+ // A non-vision model: the server persists the original image chunk AND a
+ // transcription text chunk in the SAME user message — render both.
+ const url = "data:image/png;base64,BBQ=";
+ const chunks: RenderedChunk[] = [
+ { seq: 1, role: "user", chunk: { type: "text", text: "describe this" }, provisional: false },
+ {
+ seq: 2,
+ role: "user",
+ chunk: { type: "image", url, mimeType: "image/png" },
+ provisional: false,
+ },
+ {
+ seq: 3,
+ role: "user",
+ chunk: { type: "text", text: "[Image analysis (via kimi/k2)]: a red square" },
+ provisional: false,
+ },
+ ];
+
+ const { container } = render(ChatView, { props: { chunks } });
+
+ expect(screen.getByText("describe this")).toBeInTheDocument();
+ expect(screen.getByText(/\[Image analysis/)).toBeInTheDocument();
+ expect(container.querySelector("img")?.getAttribute("src")).toBe(url);
+ });
+
+ it("renders a consult_vision tool call/result like any other tool", () => {
+ // read_image is GONE — replaced by consult_vision (opens a vision-model
+ // conversation, attaches the image + question, returns the answer). It is
+ // a normal tool call: rendered generically by toolName.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "assistant",
+ chunk: {
+ type: "tool-call",
+ toolCallId: "tc1",
+ toolName: "consult_vision",
+ input: { question: "what is in this image?", imageIds: [1] },
+ },
+ provisional: false,
+ },
+ {
+ seq: 2,
+ role: "tool",
+ chunk: {
+ type: "tool-result",
+ toolCallId: "tc1",
+ toolName: "consult_vision",
+ content: "a red square on a white background",
+ isError: false,
+ },
+ provisional: false,
+ },
+ ];
+
+ render(ChatView, { props: { chunks } });
+
+ expect(screen.getAllByText("consult_vision").length).toBeGreaterThan(0);
+ expect(screen.getByText("a red square on a white background")).toBeInTheDocument();
+ });
+
+ it("renders a non-vision placeholder text chunk as-is", () => {
+ // A non-vision model gets a numbered placeholder (a regular text chunk)
+ // instead of an auto-transcription. Renders like any text chunk.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: {
+ type: "text",
+ text: "[Image 1 attached — call consult_vision with imageIds=[1] and a specific question to analyze it]",
+ },
+ provisional: false,
+ },
+ ];
+
+ render(ChatView, { props: { chunks } });
+
+ expect(screen.getByText(/\[Image 1 attached/)).toBeInTheDocument();
+ expect(screen.getByText(/consult_vision with imageIds/)).toBeInTheDocument();
+ });
+
+ it("renders a compacted-image text chunk as-is", () => {
+ // Image compaction transcribes old images to [Compacted image]: <desc>.
+ // Regular text chunk — render as-is.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: { type: "text", text: "[Compacted image]: a chart showing rising sales" },
+ provisional: false,
+ },
+ ];
+
+ render(ChatView, { props: { chunks } });
+
+ expect(screen.getByText(/\[Compacted image\]/)).toBeInTheDocument();
+ expect(screen.getByText(/rising sales/)).toBeInTheDocument();
+ });
});
describe("Composer", () => {
@@ -623,7 +797,7 @@ describe("Composer", () => {
await user.click(sendButton);
expect(onSend).toHaveBeenCalledTimes(1);
- expect(onSend).toHaveBeenCalledWith("Hello world");
+ expect(onSend).toHaveBeenCalledWith("Hello world", undefined);
expect(textarea).toHaveValue("");
});
@@ -651,7 +825,7 @@ describe("Composer", () => {
const sendButton = screen.getByRole("button", { name: "Send" });
await user.click(sendButton);
- expect(onSend).toHaveBeenCalledWith("hello");
+ expect(onSend).toHaveBeenCalledWith("hello", undefined);
});
it("sends on Enter key (without Shift)", async () => {
@@ -663,7 +837,7 @@ describe("Composer", () => {
const textarea = screen.getByRole("textbox", { name: "Message input" });
await user.type(textarea, "Test message{Enter}");
- expect(onSend).toHaveBeenCalledWith("Test message");
+ expect(onSend).toHaveBeenCalledWith("Test message", undefined);
});
it("does not send on Shift+Enter", async () => {
@@ -677,6 +851,142 @@ describe("Composer", () => {
expect(onSend).not.toHaveBeenCalled();
});
+
+ it("stages a pasted image and forwards it on send", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "look at this");
+
+ // jsdom has no ClipboardEvent/DataTransfer: dispatch a plain paste event
+ // carrying a mock clipboardData whose only item is an image File.
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const paste = new Event("paste", { bubbles: true });
+ Object.defineProperty(paste, "clipboardData", {
+ value: {
+ items: [{ kind: "file", type: "image/png", getAsFile: () => file }],
+ },
+ });
+ container.querySelector("textarea")?.dispatchEvent(paste);
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ await user.click(screen.getByRole("button", { name: "Send" }));
+
+ expect(onSend).toHaveBeenCalledTimes(1);
+ const [, images] = onSend.mock.calls[0] ?? [];
+ expect(images).toHaveLength(1);
+ expect(images[0]?.url).toMatch(/^data:image\/png;base64,/);
+ expect(images[0]?.mimeType).toBe("image/png");
+ });
+
+ it("lets a text paste proceed when no image is on the clipboard", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "hello");
+
+ // A text-only paste: no file items → the component must NOT preventDefault,
+ // so the default text paste path is unaffected (no image staged).
+ const paste = new Event("paste", { bubbles: true });
+ Object.defineProperty(paste, "clipboardData", {
+ value: { items: [{ kind: "string", type: "text/plain" }] },
+ });
+ container.querySelector("textarea")?.dispatchEvent(paste);
+
+ await new Promise((r) => setTimeout(r, 0));
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ });
+
+ it("stages an image via the attach button's file picker", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["JPG"], "photo.jpg", { type: "image/jpeg" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ // Image-only send (no text): the Send button is enabled.
+ const send = screen.getByRole("button", { name: "Send" });
+ expect(send).not.toBeDisabled();
+ await user.click(send);
+
+ expect(onSend).toHaveBeenCalledTimes(1);
+ const [text, images] = onSend.mock.calls[0] ?? [];
+ expect(text).toBe("");
+ expect(images).toHaveLength(1);
+ expect(images[0]?.mimeType).toBe("image/jpeg");
+ });
+
+ it("removes a staged image via the remove button", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+ await user.click(screen.getByRole("button", { name: "Remove image" }));
+
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ // With no text and no images, Send is disabled again.
+ expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
+ });
+
+ it("ignores a non-image file chosen via the picker", async () => {
+ const onSend = vi.fn();
+ const { container } = render(Composer, { props: { onSend } });
+
+ const file = new File(["TXT"], "notes.txt", { type: "text/plain" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+
+ // Give the async staging a chance; a non-image is skipped.
+ await new Promise((r) => setTimeout(r, 0));
+ expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument();
+ });
+
+ it("queues (steers) text-only and never forwards images", async () => {
+ // While running, the Send button becomes "Queue"; steering is text-only.
+ const onQueue = vi.fn();
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ const { container } = render(Composer, { props: { onSend, onQueue, status: "running" } });
+
+ const textarea = screen.getByRole("textbox", { name: "Message input" });
+ await user.type(textarea, "steer here");
+
+ // Also stage an image — it must NOT be forwarded on a queue.
+ const file = new File(["PNG"], "shot.png", { type: "image/png" });
+ const input = container.querySelector('input[type="file"]') as HTMLInputElement;
+ Object.defineProperty(input, "files", { value: [file], writable: false });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+ await vi.waitFor(() => {
+ expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument();
+ });
+
+ await user.click(screen.getByRole("button", { name: "Queue" }));
+ expect(onQueue).toHaveBeenCalledWith("steer here");
+ expect(onSend).not.toHaveBeenCalled();
+ });
});
describe("ModelSelector", () => {
@@ -730,6 +1040,55 @@ describe("ModelSelector", () => {
expect(onSelect).toHaveBeenCalledTimes(1);
expect(onSelect).toHaveBeenCalledWith("openai/gpt-4o");
});
+
+ it("marks vision-capable models in the model dropdown", () => {
+ const models = ["kimi/k2", "kimi/k1.5"];
+ const modelInfo = {
+ "kimi/k2": { vision: true },
+ "kimi/k1.5": { vision: false },
+ };
+ render(ModelSelector, {
+ props: { models, selected: "kimi/k2", onSelect: vi.fn(), modelInfo },
+ });
+
+ const modelSelect = screen.getByRole("combobox", { name: "Model selector" });
+ const options = within(modelSelect).getAllByRole("option");
+ expect(options).toHaveLength(2);
+ expect(options[0]?.textContent).toContain("vision");
+ expect(options[1]?.textContent).not.toContain("vision");
+ });
+
+ it("shows the vision indicator when the selected model is vision-capable", () => {
+ render(ModelSelector, {
+ props: {
+ models: ["kimi/k2"],
+ selected: "kimi/k2",
+ onSelect: vi.fn(),
+ modelInfo: { "kimi/k2": { vision: true } },
+ },
+ });
+ expect(screen.getByText(/sees images natively/)).toBeInTheDocument();
+ });
+
+ it("shows the vision-handoff hint when the selected model is non-vision", () => {
+ render(ModelSelector, {
+ props: {
+ models: ["umans/glm-5.2"],
+ selected: "umans/glm-5.2",
+ onSelect: vi.fn(),
+ modelInfo: { "umans/glm-5.2": { vision: false } },
+ },
+ });
+ expect(screen.getByText(/auto-described/i)).toBeInTheDocument();
+ expect(screen.queryByText(/sees images natively/)).not.toBeInTheDocument();
+ });
+
+ it("shows the handoff hint when modelInfo is absent", () => {
+ render(ModelSelector, {
+ props: { models: ["openai/gpt-4"], selected: "openai/gpt-4", onSelect: vi.fn() },
+ });
+ expect(screen.getByText(/auto-described/i)).toBeInTheDocument();
+ });
});
describe("ReasoningEffortSelector", () => {
diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte
index b49f7b4..e67ca5b 100644
--- a/src/features/chat/ui/ChatView.svelte
+++ b/src/features/chat/ui/ChatView.svelte
@@ -1,6 +1,6 @@
<script lang="ts">
import type { TurnProviderRetryEvent } from "@dispatch/wire";
- import { groupRenderedChunks, type RenderedChunk, viewProviderRetry } from "../index";
+ import { groupRenderedChunks, resolveImageUrl, type RenderedChunk, viewProviderRetry } from "../index";
import {
interleaveTurnMetrics,
viewCacheRate,
@@ -24,6 +24,7 @@
onShowEarlier,
thinkingKeyBase = 0,
providerRetry = null,
+ apiBaseUrl = "",
}: {
chunks: readonly RenderedChunk[];
turnMetrics?: readonly TurnMetricsEntry[];
@@ -44,6 +45,14 @@
* newest attempt + delay, and is cleared when content resumes / turn ends.
*/
providerRetry?: TurnProviderRetryEvent | null;
+ /**
+ * The HTTP API base URL (e.g. `http://localhost:24203`). Persisted image
+ * chunks carry a compact relative path (`/images/<conv>/<uuid>.png`); this
+ * base is prepended to render them. The optimistic echo's data URL and any
+ * absolute URL pass through unchanged (see `resolveImageUrl`). Defaults to
+ * "" (root-relative — a browser resolves `/images/…` against its origin).
+ */
+ apiBaseUrl?: string;
} = $props();
// True while a show-earlier page-in is awaited (disables the button).
@@ -95,11 +104,23 @@
{#snippet chunkRow(rendered: RenderedChunk)}
{#if rendered.role === "user"}
- <!-- User: a speech bubble, left-aligned -->
+ <!-- User: a speech bubble, left-aligned. A user message may be multi-chunk
+ ([text, image, image, …]); each chunk renders in its own bubble. A
+ persisted image chunk's url is a compact relative path (`/images/…`)
+ served by the backend — resolve it against the API base. The
+ optimistic echo's data URL (and any absolute URL) passes through. -->
<div class="chat chat-start">
<div class="chat-bubble chat-bubble-primary">
{#if rendered.chunk.type === "text"}
<p>{rendered.chunk.text}</p>
+ {:else if rendered.chunk.type === "image"}
+ <img
+ src={resolveImageUrl(rendered.chunk.url, apiBaseUrl)}
+ alt={rendered.chunk.mimeType ?? "pasted image"}
+ loading="lazy"
+ decoding="async"
+ class="max-h-80 max-w-full rounded"
+ />
{/if}
</div>
</div>
diff --git a/src/features/chat/ui/Composer.svelte b/src/features/chat/ui/Composer.svelte
index 0327f93..afe1e3c 100644
--- a/src/features/chat/ui/Composer.svelte
+++ b/src/features/chat/ui/Composer.svelte
@@ -1,8 +1,19 @@
<script lang="ts">
+ import type { ImageInput } from "@dispatch/wire";
import { computeContextUsage, formatCompactTokens } from "../../../core/metrics";
const FALLBACK_CONTEXT_WINDOW = 1_000_000;
const MAX_LINES = 7;
+ /** Accept only raster images (the provider image-content formats). */
+ const IMAGE_ACCEPT = "image/png,image/jpeg,image/gif,image/webp";
+ /** Reject images larger than this before base64-encoding (keeps payloads sane). */
+ const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
+
+ /** A staged image awaiting send: a stable id + the `ImageInput` to forward. */
+ interface StagedImage {
+ readonly id: string;
+ readonly input: ImageInput;
+ }
let {
onSend,
@@ -12,12 +23,18 @@
contextWindow = undefined,
status = "idle",
}: {
- onSend: (text: string) => void;
+ /**
+ * Send a message (start a turn via `chat.send`). Carries any staged images
+ * as `ImageInput[]` (base64 data URLs or https URLs); the store forwards
+ * them on the WS `chat.send` op / `POST /chat` body. `images` is omitted
+ * (not an empty array) when none are staged, so the wire stays text-only.
+ */
+ onSend: (text: string, images?: ImageInput[]) => void;
/**
* Enqueue a steering message (`chat.queue`). When provided AND the status
* is `running`, the send button becomes a "Queue" button that steers the
- * in-flight turn instead of starting a new one. When absent, `onSend` is
- * used regardless (tests / non-steering contexts).
+ * in-flight turn instead of starting a new one. Steering is text-only —
+ * it never carries images (a mid-turn injection has no image surface).
*/
onQueue?: (text: string) => void;
/** Stop the in-flight generation (`POST /conversations/:id/stop`). */
@@ -39,22 +56,30 @@
export type ComposerStatus = "idle" | "running" | "queued" | "error";
let text = $state("");
+ let images = $state<StagedImage[]>([]);
let inputEl: HTMLTextAreaElement | undefined;
+ let fileInputEl: HTMLInputElement | undefined;
+ let dragOver = $state(false);
const hasText = $derived(text.trim().length > 0);
+ const hasImages = $derived(images.length > 0);
+ const canSend = $derived(hasText || hasImages);
const effectiveMax = $derived(contextWindow ?? FALLBACK_CONTEXT_WINDOW);
const usage = $derived(computeContextUsage(contextSize, effectiveMax));
const hasUsage = $derived(contextSize !== undefined);
// One button, three modes:
// - idle → "Send" (starts a turn via chat.send)
- // - running/queued + text → "Queue" (steers via chat.queue)
+ // - running/queued + text → "Queue" (steers via chat.queue — text only)
// - running/queued + empty → "Stop" (aborts via POST /stop)
// (`queued` behaves like `running` — the turn is in flight, just waiting for a
// concurrency slot; the user can still steer or stop it.)
+ // Steering never carries images: when running with images staged but no text,
+ // the images stay staged (queue is text-only). Images-without-text while running
+ // is an unusual case that still sends (the server auto-starts/resolves).
const inFlight = $derived(status === "running" || status === "queued");
const buttonMode = $derived.by<"send" | "queue" | "stop">(() => {
- if (inFlight && !hasText && onStop !== undefined) return "stop";
+ if (inFlight && !hasText && !hasImages && onStop !== undefined) return "stop";
if (inFlight && hasText && onQueue !== undefined) return "queue";
return "send";
});
@@ -63,7 +88,7 @@
? "Queued for a slot…"
: status === "running"
? "Steer the conversation..."
- : "Type a message...",
+ : "Type a message, paste or drop an image…",
);
// As the window fills, escalate color: calm → warning → danger.
@@ -94,15 +119,97 @@
resize();
});
+ /** Read a File into a base64 data URL (`data:image/…;base64,…`). */
+ function fileToDataUrl(file: File): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => {
+ if (typeof reader.result === "string") resolve(reader.result);
+ else reject(new Error("unreadable image"));
+ };
+ reader.onerror = () => reject(reader.error ?? new Error("read failed"));
+ reader.readAsDataURL(file);
+ });
+ }
+
+ let imgSeq = 0;
+ /** Stage a File as an image (skip non-images / oversized). Returns whether staged. */
+ async function stageFile(file: File): Promise<boolean> {
+ if (!file.type.startsWith("image/")) return false;
+ if (file.size > MAX_IMAGE_BYTES) return false;
+ const url = await fileToDataUrl(file);
+ // Prefer the file's declared MIME; fall back to the data URL's prefix.
+ const mimeType = file.type || undefined;
+ const id = `img-${Date.now()}-${imgSeq++}`;
+ images = [...images, { id, input: { url, ...(mimeType ? { mimeType } : {}) } }];
+ return true;
+ }
+
+ function removeImage(id: string): void {
+ images = images.filter((img) => img.id !== id);
+ }
+
+ /** Handle a paste anywhere in the form: extract image items from the clipboard. */
+ async function handlePaste(e: ClipboardEvent): Promise<void> {
+ const items = e.clipboardData?.items;
+ if (items === undefined) return;
+ let hadImage = false;
+ const staged: File[] = [];
+ for (const item of items) {
+ if (item.kind === "file" && item.type.startsWith("image/")) {
+ const file = item.getAsFile();
+ if (file !== null) {
+ staged.push(file);
+ hadImage = true;
+ }
+ }
+ }
+ if (!hadImage) return; // let the default text paste proceed
+ e.preventDefault(); // suppress pasting the image as a filename string
+ for (const file of staged) {
+ await stageFile(file);
+ }
+ }
+
+ /** File-picker <input type="file"> change. */
+ async function handleFilePick(e: Event): Promise<void> {
+ const target = e.currentTarget as HTMLInputElement;
+ const files = target.files;
+ if (files === null) return;
+ for (const file of files) {
+ await stageFile(file);
+ }
+ target.value = ""; // reset so picking the same file again re-fires change
+ }
+
+ /** Drop images onto the composer. */
+ async function handleDrop(e: DragEvent): Promise<void> {
+ dragOver = false;
+ const files = e.dataTransfer?.files;
+ if (files === undefined || files.length === 0) return;
+ const hadImage = Array.from(files).some((f) => f.type.startsWith("image/"));
+ if (!hadImage) return;
+ e.preventDefault();
+ for (const file of files) {
+ await stageFile(file);
+ }
+ }
+
function handleSubmit(): void {
const trimmed = text.trim();
- if (trimmed.length === 0) return;
+ // Allow a send with images even when text is empty (an image-only turn).
+ if (trimmed.length === 0 && !hasImages) return;
if (buttonMode === "queue") {
+ // Steering is text-only — never forward images.
onQueue?.(trimmed);
} else {
- onSend(trimmed);
+ const toSend: ImageInput[] | undefined = hasImages
+ ? images.map((img) => img.input)
+ : undefined;
+ onSend(trimmed, toSend);
}
text = "";
+ images = [];
}
function handleKeydown(e: KeyboardEvent): void {
@@ -119,17 +226,68 @@
e.preventDefault();
handleSubmit();
}}
+ ondrop={handleDrop}
+ ondragover={(e) => {
+ if (e.dataTransfer?.types?.includes("Files")) {
+ e.preventDefault();
+ dragOver = true;
+ }
+ }}
+ ondragleave={() => (dragOver = false)}
>
- <!-- Top bar: expanding textarea + single context-aware button -->
+ <!-- Top bar: expanding textarea + image-attach button + single context-aware button -->
<div class="flex items-end gap-2 px-4 pt-3 pb-2">
- <textarea
- bind:this={inputEl}
- class="textarea textarea-bordered flex-1 resize-none leading-normal !min-h-0 h-auto"
- bind:value={text}
- onkeydown={handleKeydown}
- {placeholder}
- rows="1"
- aria-label="Message input"></textarea>
+ <div
+ class="flex-1"
+ onpaste={handlePaste}
+ class:border-2={dragOver}
+ class:border-primary={dragOver}
+ class:border-dashed={dragOver}
+ class:rounded={dragOver}
+ >
+ <textarea
+ bind:this={inputEl}
+ class="textarea textarea-bordered w-full resize-none leading-normal !min-h-0 h-auto"
+ bind:value={text}
+ onkeydown={handleKeydown}
+ {placeholder}
+ rows="1"
+ aria-label="Message input"></textarea>
+ </div>
+
+ <!-- Hidden file picker (images only; multiple). -->
+ <input
+ bind:this={fileInputEl}
+ type="file"
+ accept={IMAGE_ACCEPT}
+ multiple
+ class="hidden"
+ onchange={handleFilePick}
+ />
+ <!-- Attach image button (opens the file picker). -->
+ <button
+ class="btn btn-ghost btn-square shrink-0"
+ type="button"
+ aria-label="Attach image"
+ title="Attach image"
+ onclick={() => fileInputEl?.click()}
+ >
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ stroke-width="2"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ class="h-5 w-5"
+ >
+ <rect x="3" y="3" width="18" height="18" rx="2" ry="2"></rect>
+ <circle cx="8.5" cy="8.5" r="1.5"></circle>
+ <polyline points="21 15 16 10 5 21"></polyline>
+ </svg>
+ </button>
+
{#if buttonMode === "stop"}
<button
class="btn btn-error w-20 shrink-0"
@@ -140,12 +298,47 @@
Stop
</button>
{:else}
- <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!hasText}>
+ <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!canSend}>
{buttonMode === "queue" ? "Queue" : "Send"}
</button>
{/if}
</div>
+ <!-- Staged image thumbnails (previews) with remove buttons. -->
+ {#if hasImages}
+ <div class="flex flex-wrap gap-2 px-4 pb-1">
+ {#each images as img (img.id)}
+ <div class="group relative h-20 w-20 shrink-0 overflow-hidden rounded border border-base-300">
+ <img
+ src={img.input.url}
+ alt={img.input.mimeType ?? "staged image"}
+ class="h-full w-full object-cover"
+ />
+ <button
+ class="btn btn-circle btn-xs absolute right-0 top-0 bg-base-100/80 hover:bg-error hover:text-error-content"
+ type="button"
+ aria-label="Remove image"
+ onclick={() => removeImage(img.id)}
+ >
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ stroke-width="3"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ class="h-3 w-3"
+ >
+ <line x1="18" y1="6" x2="6" y2="18"></line>
+ <line x1="6" y1="6" x2="18" y2="18"></line>
+ </svg>
+ </button>
+ </div>
+ {/each}
+ </div>
+ {/if}
+
<!-- Bottom status bar: status icon · context-window fill · token count -->
<div class="flex items-center gap-2 px-4 pb-2 text-xs text-base-content/50">
<span class="shrink-0">
diff --git a/src/features/chat/ui/ModelSelector.svelte b/src/features/chat/ui/ModelSelector.svelte
index 03acb79..11b9feb 100644
--- a/src/features/chat/ui/ModelSelector.svelte
+++ b/src/features/chat/ui/ModelSelector.svelte
@@ -1,20 +1,32 @@
<script lang="ts">
- import { joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select";
+ import type { ModelMetadata } from "@dispatch/transport-contract";
+ import { isVisionModel, joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select";
let {
models,
selected,
onSelect,
+ modelInfo = {},
}: {
models: readonly string[];
selected: string;
onSelect: (model: string) => void;
+ /**
+ * Per-model metadata from `GET /models` (`{ [name]: ModelMetadata }`).
+ * Used to show a "vision" badge next to models with `vision: true` (they
+ * natively accept images; others rely on the server's vision handoff).
+ * Optional — absent metadata → no badge (treated as non-vision).
+ */
+ modelInfo?: Readonly<Record<string, ModelMetadata>>;
} = $props();
const keys = $derived(modelKeys(models));
const current = $derived(splitModelName(selected));
const keyModels = $derived(modelsForKey(models, current.key));
+ // Whether the currently-selected full model name is vision-capable.
+ const selectedVision = $derived(isVisionModel(modelInfo, selected));
+
// Switching key jumps to the first model available under it.
function selectKey(key: string): void {
const first = modelsForKey(models, key)[0] ?? "";
@@ -24,6 +36,11 @@
function selectModel(model: string): void {
onSelect(joinModelName(current.key, model));
}
+
+ // The full `<key>/<model>` name for a model suffix under the current key.
+ function fullNameFor(modelSuffix: string): string {
+ return joinModelName(current.key, modelSuffix);
+ }
</script>
<div class="flex flex-col gap-2">
@@ -44,7 +61,32 @@
aria-label="Model selector"
>
{#each keyModels as model (model)}
- <option value={model}>{model}</option>
+ <option value={model}>
+ {model}{#if isVisionModel(modelInfo, fullNameFor(model))} · vision{/if}
+ </option>
{/each}
</select>
+ {#if selectedVision}
+ <div class="flex items-center gap-1 text-xs text-base-content/60">
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ stroke-width="2"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ class="h-3.5 w-3.5"
+ aria-hidden="true"
+ >
+ <path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"></path>
+ <circle cx="12" cy="12" r="3"></circle>
+ </svg>
+ <span>Vision — this model sees images natively</span>
+ </div>
+ {:else}
+ <div class="text-xs text-base-content/40">
+ Pasted images are auto-described (vision handoff)
+ </div>
+ {/if}
</div>