diff options
| author | Adam Malczewski <[email protected]> | 2026-06-27 19:15:04 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-27 19:15:04 +0900 |
| commit | fa0bd9c0e433b1abddc814b48a358c94954c7d36 (patch) | |
| tree | dedf7adf827de6908b5c7cbb4b4153aba38c1882 /src/features/chat | |
| parent | 3566a20ebbded754070fce66af48690d1a904879 (diff) | |
| download | dispatch-web-fa0bd9c0e433b1abddc814b48a358c94954c7d36.tar.gz dispatch-web-fa0bd9c0e433b1abddc814b48a358c94954c7d36.zip | |
feat(vision): consult_vision tool + vision settings API
Backend vision update (additive to [email protected] / [email protected],
no version bump).
Contracts mirrored (.dispatch/transport-contract.reference.md):
- VisionSettingsResponse + SetVisionSettingsRequest (GET/PUT /settings/vision).
- Delta note: read_image -> consult_vision; numbered placeholders; compaction.
Tool rendering (ChatView):
- read_image test -> consult_vision (rendering is generic by toolName).
- +2 tests: numbered-placeholder text chunk + [Compacted image] text chunk
(both regular text chunks, render as-is — no special handling).
New vision feature library (src/features/vision/):
- logic/view-model.ts (32 tests): VisionSettings/VisionSettingsPatch types
(consumer-defines-port), LoadVisionSettings/SaveVisionSettings ports +
results, normalizeVisionSettings (network-seam coercion), parseImageLimit/
imageLimitChanged, compactionModelOptions (vision-capable models via chat's
public isVisionModel + Auto sentinel), round-trip helpers, imageLimitLabel.
- ui/VisionSettingsView.svelte (9 tests): imageLimit input + Save,
compactionModel dropdown (Auto + vision-capable models), load-on-mount,
save-on-change, error/saved feedback.
- index.ts.
Cross-unit seam: isVisionModel added to features/chat public index.ts
(additive); imported through the public surface, not internals.
Store wiring (src/app/store.svelte.ts):
- visionSettings state + refreshVisionSettings (GET /settings/vision,
normalized at the seam) + setVisionSettings (PUT, partial, returns merged) +
VisionSettingsResult; seeded on boot; exposed on AppStore. +4 store tests.
Mounted in App.svelte: new "Vision" sidebar view kind + VisionSettingsView
in viewContent (not conversation-scoped); load/save adapters; visionManifest.
Verification: svelte-check 0/0; vitest 948/948 (run twice, +47 since the
prior vision commit); biome clean; vite build OK. See backend-handoff.md §2j.
Not merged or pushed.
Diffstat (limited to 'src/features/chat')
| -rw-r--r-- | src/features/chat/index.ts | 1 | ||||
| -rw-r--r-- | src/features/chat/ui.test.ts | 57 |
2 files changed, 50 insertions, 8 deletions
diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts index ddb094d..8694691 100644 --- a/src/features/chat/index.ts +++ b/src/features/chat/index.ts @@ -6,6 +6,7 @@ export type { } from "../../core/chunks"; export { groupRenderedChunks, viewProviderRetry } from "../../core/chunks"; export type { TurnMetricsEntry } from "../../core/metrics"; +export { isVisionModel } from "./model-select"; export type { ChatTransport, HistorySync, HistoryWindow, MetricsSync } from "./ports"; export type { EffortOption, diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index af32f72..b0aa6f0 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -655,8 +655,10 @@ describe("ChatView", () => { expect(container.querySelector("img")?.getAttribute("src")).toBe(url); }); - it("renders a read_image tool call/result like any other tool", () => { - // The read_image tool is available to all models; it is a normal tool call. + it("renders a consult_vision tool call/result like any other tool", () => { + // read_image is GONE — replaced by consult_vision (opens a vision-model + // conversation, attaches the image + question, returns the answer). It is + // a normal tool call: rendered generically by toolName. const chunks: RenderedChunk[] = [ { seq: 1, @@ -664,8 +666,8 @@ describe("ChatView", () => { chunk: { type: "tool-call", toolCallId: "tc1", - toolName: "read_image", - input: { path: "/tmp/screenshot.png" }, + toolName: "consult_vision", + input: { question: "what is in this image?", imageIds: [1] }, }, provisional: false, }, @@ -675,8 +677,8 @@ describe("ChatView", () => { chunk: { type: "tool-result", toolCallId: "tc1", - toolName: "read_image", - content: "a screenshot of the desktop", + toolName: "consult_vision", + content: "a red square on a white background", isError: false, }, provisional: false, @@ -685,8 +687,47 @@ describe("ChatView", () => { render(ChatView, { props: { chunks } }); - expect(screen.getAllByText("read_image").length).toBeGreaterThan(0); - expect(screen.getByText("a screenshot of the desktop")).toBeInTheDocument(); + expect(screen.getAllByText("consult_vision").length).toBeGreaterThan(0); + expect(screen.getByText("a red square on a white background")).toBeInTheDocument(); + }); + + it("renders a non-vision placeholder text chunk as-is", () => { + // A non-vision model gets a numbered placeholder (a regular text chunk) + // instead of an auto-transcription. Renders like any text chunk. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { + type: "text", + text: "[Image 1 attached — call consult_vision with imageIds=[1] and a specific question to analyze it]", + }, + provisional: false, + }, + ]; + + render(ChatView, { props: { chunks } }); + + expect(screen.getByText(/\[Image 1 attached/)).toBeInTheDocument(); + expect(screen.getByText(/consult_vision with imageIds/)).toBeInTheDocument(); + }); + + it("renders a compacted-image text chunk as-is", () => { + // Image compaction transcribes old images to [Compacted image]: <desc>. + // Regular text chunk — render as-is. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "text", text: "[Compacted image]: a chart showing rising sales" }, + provisional: false, + }, + ]; + + render(ChatView, { props: { chunks } }); + + expect(screen.getByText(/\[Compacted image\]/)).toBeInTheDocument(); + expect(screen.getByText(/rising sales/)).toBeInTheDocument(); }); }); |
