summaryrefslogtreecommitdiffhomepage
path: root/src/features/chat
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-27 19:15:04 +0900
committerAdam Malczewski <[email protected]>2026-06-27 19:15:04 +0900
commitfa0bd9c0e433b1abddc814b48a358c94954c7d36 (patch)
treededf7adf827de6908b5c7cbb4b4153aba38c1882 /src/features/chat
parent3566a20ebbded754070fce66af48690d1a904879 (diff)
downloaddispatch-web-fa0bd9c0e433b1abddc814b48a358c94954c7d36.tar.gz
dispatch-web-fa0bd9c0e433b1abddc814b48a358c94954c7d36.zip
feat(vision): consult_vision tool + vision settings API
Backend vision update (additive to [email protected] / [email protected], no version bump). Contracts mirrored (.dispatch/transport-contract.reference.md): - VisionSettingsResponse + SetVisionSettingsRequest (GET/PUT /settings/vision). - Delta note: read_image -> consult_vision; numbered placeholders; compaction. Tool rendering (ChatView): - read_image test -> consult_vision (rendering is generic by toolName). - +2 tests: numbered-placeholder text chunk + [Compacted image] text chunk (both regular text chunks, render as-is — no special handling). New vision feature library (src/features/vision/): - logic/view-model.ts (32 tests): VisionSettings/VisionSettingsPatch types (consumer-defines-port), LoadVisionSettings/SaveVisionSettings ports + results, normalizeVisionSettings (network-seam coercion), parseImageLimit/ imageLimitChanged, compactionModelOptions (vision-capable models via chat's public isVisionModel + Auto sentinel), round-trip helpers, imageLimitLabel. - ui/VisionSettingsView.svelte (9 tests): imageLimit input + Save, compactionModel dropdown (Auto + vision-capable models), load-on-mount, save-on-change, error/saved feedback. - index.ts. Cross-unit seam: isVisionModel added to features/chat public index.ts (additive); imported through the public surface, not internals. Store wiring (src/app/store.svelte.ts): - visionSettings state + refreshVisionSettings (GET /settings/vision, normalized at the seam) + setVisionSettings (PUT, partial, returns merged) + VisionSettingsResult; seeded on boot; exposed on AppStore. +4 store tests. Mounted in App.svelte: new "Vision" sidebar view kind + VisionSettingsView in viewContent (not conversation-scoped); load/save adapters; visionManifest. Verification: svelte-check 0/0; vitest 948/948 (run twice, +47 since the prior vision commit); biome clean; vite build OK. See backend-handoff.md §2j. Not merged or pushed.
Diffstat (limited to 'src/features/chat')
-rw-r--r--src/features/chat/index.ts1
-rw-r--r--src/features/chat/ui.test.ts57
2 files changed, 50 insertions, 8 deletions
diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts
index ddb094d..8694691 100644
--- a/src/features/chat/index.ts
+++ b/src/features/chat/index.ts
@@ -6,6 +6,7 @@ export type {
} from "../../core/chunks";
export { groupRenderedChunks, viewProviderRetry } from "../../core/chunks";
export type { TurnMetricsEntry } from "../../core/metrics";
+export { isVisionModel } from "./model-select";
export type { ChatTransport, HistorySync, HistoryWindow, MetricsSync } from "./ports";
export type {
EffortOption,
diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts
index af32f72..b0aa6f0 100644
--- a/src/features/chat/ui.test.ts
+++ b/src/features/chat/ui.test.ts
@@ -655,8 +655,10 @@ describe("ChatView", () => {
expect(container.querySelector("img")?.getAttribute("src")).toBe(url);
});
- it("renders a read_image tool call/result like any other tool", () => {
- // The read_image tool is available to all models; it is a normal tool call.
+ it("renders a consult_vision tool call/result like any other tool", () => {
+ // read_image is GONE — replaced by consult_vision (opens a vision-model
+ // conversation, attaches the image + question, returns the answer). It is
+ // a normal tool call: rendered generically by toolName.
const chunks: RenderedChunk[] = [
{
seq: 1,
@@ -664,8 +666,8 @@ describe("ChatView", () => {
chunk: {
type: "tool-call",
toolCallId: "tc1",
- toolName: "read_image",
- input: { path: "/tmp/screenshot.png" },
+ toolName: "consult_vision",
+ input: { question: "what is in this image?", imageIds: [1] },
},
provisional: false,
},
@@ -675,8 +677,8 @@ describe("ChatView", () => {
chunk: {
type: "tool-result",
toolCallId: "tc1",
- toolName: "read_image",
- content: "a screenshot of the desktop",
+ toolName: "consult_vision",
+ content: "a red square on a white background",
isError: false,
},
provisional: false,
@@ -685,8 +687,47 @@ describe("ChatView", () => {
render(ChatView, { props: { chunks } });
- expect(screen.getAllByText("read_image").length).toBeGreaterThan(0);
- expect(screen.getByText("a screenshot of the desktop")).toBeInTheDocument();
+ expect(screen.getAllByText("consult_vision").length).toBeGreaterThan(0);
+ expect(screen.getByText("a red square on a white background")).toBeInTheDocument();
+ });
+
+ it("renders a non-vision placeholder text chunk as-is", () => {
+ // A non-vision model gets a numbered placeholder (a regular text chunk)
+ // instead of an auto-transcription. Renders like any text chunk.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: {
+ type: "text",
+ text: "[Image 1 attached — call consult_vision with imageIds=[1] and a specific question to analyze it]",
+ },
+ provisional: false,
+ },
+ ];
+
+ render(ChatView, { props: { chunks } });
+
+ expect(screen.getByText(/\[Image 1 attached/)).toBeInTheDocument();
+ expect(screen.getByText(/consult_vision with imageIds/)).toBeInTheDocument();
+ });
+
+ it("renders a compacted-image text chunk as-is", () => {
+ // Image compaction transcribes old images to [Compacted image]: <desc>.
+ // Regular text chunk — render as-is.
+ const chunks: RenderedChunk[] = [
+ {
+ seq: 1,
+ role: "user",
+ chunk: { type: "text", text: "[Compacted image]: a chart showing rising sales" },
+ provisional: false,
+ },
+ ];
+
+ render(ChatView, { props: { chunks } });
+
+ expect(screen.getByText(/\[Compacted image\]/)).toBeInTheDocument();
+ expect(screen.getByText(/rising sales/)).toBeInTheDocument();
});
});