diff options
| author | Adam Malczewski <[email protected]> | 2026-06-27 20:49:43 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-27 20:49:43 +0900 |
| commit | a59200e786f7d97d7ba5b9cd2bee9ffef531dac2 (patch) | |
| tree | 92088287487e34a9fee37f89325961110b1d3442 | |
| parent | a9ca756de8cd023c0f2cb9954f344fff11146bc2 (diff) | |
| parent | b70ae547fdcb8c1794981957485537dc21a8b5fd (diff) | |
| download | dispatch-web-a59200e786f7d97d7ba5b9cd2bee9ffef531dac2.tar.gz dispatch-web-a59200e786f7d97d7ba5b9cd2bee9ffef531dac2.zip | |
Merge branch 'feature/vision-handoff' into dev
# Conflicts:
# .dispatch/transport-contract.reference.md
# backend-handoff.md
# src/app/App.svelte
# src/features/chat/ui/Composer.svelte
29 files changed, 2675 insertions, 72 deletions
diff --git a/.dispatch/transport-contract.reference.md b/.dispatch/transport-contract.reference.md index 3522cbb..6a5a6b3 100644 --- a/.dispatch/transport-contract.reference.md +++ b/.dispatch/transport-contract.reference.md @@ -15,6 +15,32 @@ > `concurrency` extension tracks/limits in-flight token-generating requests per provider with > oldest-agent-first queueing; when it isn't loaded the list + status endpoints return empty arrays and > the single/PUT/DELETE return `503`. See `backend-handoff.md` §2j. +> **2026-06-26 delta (vision handoff — ADDITIVE, NO version bump):** adds the vision/image surface. +> `ChatRequest` (+ `ChatSendMessage`/`QueueRequest`) gains an optional `images?: readonly ImageInput[]` +> (each entry: `{ url, mimeType? }` — a base64 data URL or `http(s)://` URL; validated non-array/no-url/ +> empty-url → 400, empty array treated as absent). `ModelMetadata` gains `vision?: boolean` (true when the +> model natively accepts images; absent → the server's vision handoff transcribes images to text before the +> model sees them). `ImageChunk`/`ImageInput` are `@dispatch/wire` types (re-exported here). +> +> **2026-06-26 update (consult_vision + vision settings — ADDITIVE, NO version bump):** the `read_image` +> tool is REPLACED by `consult_vision` (`{ question: string, imageIds?: number[], path?: string }`) — it +> opens a NEW conversation tab with a vision-capable model, attaches the image + question, and returns the +> vision model's answer (rendered like any tool call/result). Non-vision models now get NUMBERED +> PLACEHOLDERS (`[Image N attached — call consult_vision with imageIds=[N] and a specific question to +> analyze it]`) instead of auto-transcriptions — these are regular `text` chunks (render as-is). Image +> compaction transcribes the oldest images past `imageLimit` to `[Compacted image]: <description>` text +> chunks (also regular `text` — render as-is; the persisted `image` chunk stays for rendering). NEW global +> vision settings API: `GET /settings/vision` → `VisionSettingsResponse` (`{ imageLimit, compactionModel }`), +> `PUT /settings/vision` ← `SetVisionSettingsRequest` (partial: `imageLimit?` non-negative int, 0 = disable +> compaction; `compactionModel?` `<key>/<model>` or null = auto). See `backend-handoff.md` §2j. +> +> **2026-06-26 update (image storage — NO type change, behavior only):** persisted `ImageChunk.url`s are now +> compact relative HTTP paths (`/images/<conversationId>/<uuid>.png`) served by the new +> `GET /images/:conversationId/:imageId` endpoint (raw image bytes + correct Content-Type) — NOT base64 data +> URLs (images are stored on disk under tmp, not in the SQLite store). `ChatRequest.images` (`ImageInput.url`) +> is UNCHANGED — clients still send data URLs; the backend saves them to tmp and returns compact paths in +> the persisted chunks. A client resolves a relative `url` against its API base (`resolveImageUrl`); the +> optimistic echo's data URL and any absolute URL pass through. See `backend-handoff.md` §2j. > > **2026-06-25 delta (SSH handoff #2 — ADDITIVE to `[email protected]`, NO version bump):** adds the > computer HTTP API types: `ComputerListResponse` (`GET /computers`), `ComputerResponse` (`GET /computers/:alias`), @@ -67,8 +93,11 @@ import type { SurfaceClientMessage, SurfaceServerMessage } from "@dispatch/ui-contract"; import type { AgentEvent, + Computer, + ComputerEntry, ConversationMeta, ConversationStatus, + ImageInput, QueuedMessage, ReasoningEffort, StoredChunk, @@ -80,8 +109,12 @@ import type { export type { AgentEvent, CompactionResult, + Computer, + ComputerEntry, ConversationMeta, ConversationStatus, + ImageChunk, + ImageInput, QueuedMessage, ReasoningEffort, StepMetrics, @@ -109,6 +142,21 @@ export interface ChatRequest { readonly message: string; /** + * Images attached to this turn (e.g. a user-pasted screenshot). Each entry's + * `url` is a base64 data URL (`data:image/…;base64,…`) or an `http(s)://` + * URL. The server converts these to `image` chunks on the persisted user + * message. For a VISION-capable model (e.g. kimi), the images are passed + * through to the provider natively. For a NON-vision model (e.g. glm-5.2), + * the server's vision handoff transcribes each image to a text description + * (via a vision-capable model) and feeds that text instead — so a text-only + * model can still reason about the image's contents. Optional — omit for a + * text-only turn (backward compatible). Validation: non-array `images` → + * 400; an image without `url` → 400; empty `url` → 400. An empty array is + * accepted and treated as absent. + */ + readonly images?: readonly ImageInput[]; + + /** * The model to use, as a model name in `<credentialName>/<model>` form — one * of the exact strings returned by `GET /models`. Omit to use the server's * default credential + model. @@ -165,6 +213,14 @@ export interface ModelsResponse { /** Per-model metadata returned alongside the model catalog. */ export interface ModelMetadata { readonly contextWindow?: number; + /** + * Whether this model can natively accept image input (vision/multimodal). + * When `true`, image chunks in a user message are passed through to the + * provider. When `false`/absent, the server's vision handoff transcribes + * images to text before the model sees them. A client may use this to show + * a vision badge in the model picker. Optional — absent when unknown. + */ + readonly vision?: boolean; } /** @@ -428,6 +484,27 @@ export interface SystemPromptVariablesResponse { readonly variables: readonly SystemPromptVariable[]; } +// ─── Vision settings (global) ─────────────────────────────────────────────── + +/** + * Response of `GET /settings/vision` — the global vision configuration shared + * across all conversations and vision models. + */ +export interface VisionSettingsResponse { + /** Max native images per turn (default 10); 0 disables image compaction. */ + readonly imageLimit: number; + /** Which model transcribes old images (null = auto-select a vision model). */ + readonly compactionModel: string | null; +} + +/** Body of `PUT /settings/vision` — a partial update. */ +export interface SetVisionSettingsRequest { + /** Non-negative integer (0 = disable compaction). */ + readonly imageLimit?: number; + /** A model name (`<key>/<model>`) or null (auto). */ + readonly compactionModel?: string | null; +} + // ─── Message queue (steering) ───────────────────────────────────────────────── /** diff --git a/.dispatch/wire.reference.md b/.dispatch/wire.reference.md index f9a14f8..888c160 100644 --- a/.dispatch/wire.reference.md +++ b/.dispatch/wire.reference.md @@ -11,6 +11,23 @@ > per-provider concurrency slot (broadcast-only via `conversation.statusChanged`, never persisted); the FE shows > a loading ring (vs the dots of `active`). See `backend-handoff.md` CR-13. > +> **2026-06-26 delta (vision handoff — ADDITIVE to `[email protected]`, NO version bump):** adds a new +> `ImageChunk` variant to the `Chunk` union (`{ type: "image", url, mimeType? }` — `url` is a base64 data +> URL or an `http(s)://` URL) and a transport-facing `ImageInput` (`{ url, mimeType? }`, what a client +> sends on `ChatRequest.images`; the orchestrator converts each into an `ImageChunk` on the persisted user +> message). Vision-capable models receive image chunks natively; non-vision models never see them directly +> — the orchestrator's vision handoff transcribes each to a text description (persisted as a separate +> `text` chunk in the SAME user message). See `backend-handoff.md` §2j. +> +> **2026-06-26 update (image storage — NO type change, behavior only):** `ImageChunk.url` for PERSISTED +> chunks is now a compact relative HTTP path (`/images/<conversationId>/<uuid>.png`) served by the backend's +> new `GET /images/:conversationId/:imageId` endpoint (raw bytes + correct Content-Type), NOT a base64 data +> URL — images are stored on disk under tmp, not in the SQLite conversation store (keeps payloads small). +> `ImageInput.url` (what a client SENDS on `ChatRequest.images`) is UNCHANGED — still a data URL or +> `http(s)://` URL; the backend saves it to tmp and returns the compact path in the persisted chunk. A client +> resolves a relative `url` against its API base (`resolveImageUrl`); a data URL (the optimistic echo) or an +> absolute URL passes through unchanged. See `backend-handoff.md` §2j. +> > **2026-06-23 delta (workspaces handoff — package bumped `0.11.0` → `0.12.0`, ADDITIVE):** adds > `Workspace` + `WorkspaceEntry` (a list entry with a conversation count) and a required > `workspaceId: string` on `ConversationMeta` (`"default"` for legacy/unspecified conversations). A @@ -73,7 +90,8 @@ export type Chunk = | ToolCallChunk | ToolResultChunk | ErrorChunk - | SystemChunk; + | SystemChunk + | ImageChunk; /** A piece of plain text content from the assistant or user. */ export interface TextChunk { @@ -150,6 +168,51 @@ export interface SystemChunk { } /** + * An image attached to a message (e.g. a user-pasted screenshot or pasted + * photo). Carries a `url` that is EITHER a base64 data URL + * (`data:image/png;base64,…`) OR an `http(s)://` URL OR — for PERSISTED chunks + * (history/replay) — a compact relative HTTP path (`/images/<conversationId>/ + * <uuid>.png`) served by the backend's `GET /images/:conversationId/:imageId` + * endpoint (images are stored on disk under tmp, NOT in the conversation store, + * to keep SQLite payloads small). A client resolves a relative path against its + * API base URL; a data URL (the optimistic echo / a pasted image) or an + * absolute URL is rendered as-is. Vision-capable models receive it natively + * (the provider serializes it to its image-content format); non-vision models + * never see it directly — the orchestrator's **vision handoff** transcribes it + * to a text description (via a vision-capable model) and feeds that text + * instead, so a text-only model can still reason about the image's contents. + * + * When a transcription was performed, it is persisted as a separate `text` + * chunk alongside the `image` chunk in the SAME user message, so the + * description is reused on every later turn (no re-transcription) and a + * client renders both the original image and its textual analysis. + */ +export interface ImageChunk { + readonly type: "image"; + /** Image source: a base64 data URL (`data:image/…;base64,…`), an `http(s)://` URL, or a compact relative path (`/images/<conv>/<uuid>.png`) for persisted chunks. */ + readonly url: string; + /** + * Optional MIME type of the image (e.g. `"image/png"`). Inferred from the + * data URL when absent; present so a client can render an icon/label without + * parsing the URL. Optional — callers that only have a URL omit it. + */ + readonly mimeType?: string; +} + +/** + * An image a client attaches to a chat message (`ChatRequest.images`). The + * transport-facing input shape; the orchestrator converts each `ImageInput` + * into an `ImageChunk` on the persisted user message. Carries the same `url` + * semantics as `ImageChunk.url`. + */ +export interface ImageInput { + /** Image source: a base64 data URL (`data:image/…;base64,…`) or an `http(s)://` URL. */ + readonly url: string; + /** Optional MIME type (e.g. `"image/png"`). Optional — inferred from the data URL when absent. */ + readonly mimeType?: string; +} + +/** * A chat message: a role plus an ordered sequence of chunks. Messages are the * unit passed to and from the provider; chunks are the unit persisted and * rendered. diff --git a/GLOSSARY.md b/GLOSSARY.md index f0dcdd4..14b81d1 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -26,6 +26,10 @@ | **steering** | A user message injected into an in-flight turn at the tool-result boundary (drawn from the **message queue**): the model sees it alongside the tool results and may adjust course. Emitted on the chat stream as a `steering` `AgentEvent` (`TurnSteeringEvent`); the queue surface clears on drain (move, don't duplicate). If the turn ends with a non-empty queue (no tool call fired), the queue carries into a NEW turn as its opening prompt (no `steering` event). `[email protected]`. | mid-turn injection, course correction, interruption | | **computer** | A remote SSH target discovered from the system's `~/.ssh/config` — a read-only VIEW, NOT an editable entity (no CRUD store; to add one the user edits `~/.ssh/config`). Backend-canonical (`[email protected]`, additive). On the wire as `Computer` (`{ alias, hostName, port, user, identityFile, knownHost }`) + `ComputerEntry extends Computer` (adds `usageCount`, for `GET /computers`). `alias` IS the **computerId** — the string persisted per-conversation / per-workspace (the computer analog of `cwd`). Resolution is SERVER-owned (never re-implement): per-conversation `computerId` → `workspace.defaultComputerId` → `null`/local. USER-facing only: a tool-execution target forwarded to tools, NEVER part of the model prompt (does not affect prompt caching); the agent never sees it. HTTP API (`GET /computers`, `GET`/`PUT /conversations/:id/computer`, `PUT /workspaces/:id/default-computer`, `GET /computers/:alias/status`, `POST /computers/:alias/test`) consumed in handoff #2. | ssh host, remote host, server, connection target | | **computerId** | The string id of a **computer** — an SSH config `Host` alias users select. Persisted per-conversation and per-workspace (the computer analog of `cwd`/`workspaceId`). `null` means local (no SSH; today's behavior). On `Workspace` as the REQUIRED `defaultComputerId: string \| null` (null = local / no SSH; the computer analog of `defaultCwd`); per-conversation persistence via `GET`/`PUT`/`DELETE /conversations/:id/computer`. `chat.send` need not send it (resolved server-side from the persisted per-conversation value in the MVP). | ssh alias, host id, remote id | +| **image chunk** | An `ImageChunk` (`{ type: "image", url, mimeType? }`) — a `Chunk` variant for an image attached to a message (a user-pasted screenshot/photo). `url` is a base64 data URL (`data:image/…;base64,…`), an `http(s)://` URL, or — for PERSISTED chunks — a compact relative HTTP path (`/images/<conversationId>/<uuid>.png`) served by `GET /images/:conversationId/:imageId` (images are stored on disk under tmp, not in the SQLite store). Backend-canonical (`[email protected]`, additive). A user message may be multi-chunk (`[text, image, image, …]` in order). On the wire as `ImageChunk` (persisted) + `ImageInput` (what `ChatRequest.images` carries — still a data URL; the orchestrator saves it to tmp and returns the compact path). A client resolves a relative `url` against its API base (`resolveImageUrl`); a data URL (the optimistic echo) or absolute URL passes through. | picture, photo attachment, screenshot chunk | +| **vision** (capability) | Whether a model can natively accept image input (multimodal). On the wire as `ModelMetadata.vision?: boolean` (`GET /models` `modelInfo[name].vision`). `true` (e.g. any `kimi/*` model) → image chunks are passed through to the provider natively. Absent/`false` (e.g. `umans/glm-5.2`) → the server's **vision handoff** gives the model a numbered placeholder + the `consult_vision` tool. The FE shows a vision badge in the model picker; it does NOT decide handoff (server-owned). | multimodal, image support | +| **vision handoff** | The server-owned mechanism by which a NON-vision model still reasons about a pasted image. A non-vision model gets a NUMBERED PLACEHOLDER text chunk alongside the persisted `image` chunk, then calls the `consult_vision` tool (which opens a NEW conversation tab with a vision-capable model, attaches the image + question, and returns the vision model's answer). The persisted user message keeps the original `image` chunk (the FE renders it) AND the placeholder text. When a vision-capable model has more than `imageLimit` images in history, image **compaction** transcribes the oldest to `[Compacted image]: <description>` text chunks (the `image` chunk stays for rendering). All these are regular `text` chunks — the FE renders them as-is. Distinct from a vision-capable model, which receives the image natively. | image transcription, vision relay | +| **consult_vision** | A tool available to all models that defers image analysis to a vision-capable model: it opens a NEW conversation tab (with a vision model, e.g. Kimi), attaches the image (by `imageIds` from a pasted placeholder, or by `path` from disk) + a `question`, and returns the conversation id + the vision model's answer (suggesting the dispatch CLI for follow-ups). Replaces the former `read_image` tool. Rendered generically like any tool call/result (by `toolName`). | read_image (former), vision tool | ## Frontend-specific | Term | Meaning | Aliases to avoid | diff --git a/backend-handoff.md b/backend-handoff.md index 67edf1a..5b41741 100644 --- a/backend-handoff.md +++ b/backend-handoff.md @@ -9,6 +9,12 @@ _Last updated: 2026-06-26 (backend: concurrency limits now PERSISTED across rebo re-pin/re-mirror needed; §2j updated. FE: brief "Saved." confirmation on the limit row after a successful save. 926 tests green.) Prior: CR-13 (`"queued"` ConversationStatus) RESOLVED; dev merged (e81df4c)._ **FE is current on `[email protected]` / `[email protected]` / `[email protected]`.** Open asks: **CR-9** +_Last updated: 2026-06-26 (§2j UPDATED — Image storage: persisted `ImageChunk.url`s are now compact relative HTTP +paths (`/images/<conv>/<uuid>.png`) served by `GET /images/:conversationId/:imageId` (images stored on disk under tmp, +not SQLite). FE resolves relative urls against the API base via a new pure `resolveImageUrl` helper; the optimistic +echo's data URL passes through unchanged; `ChatRequest.images` (send) is unchanged. typecheck 0/0, 959 tests green +(+11), biome clean, build OK. §2i unchanged.)_ +**FE is current on `[email protected]` / `[email protected]` / `[email protected]`.** Open asks: **CR-9** (`system:os` should detect WSL + include Linux distro — backend behavior change, no contract bump). The SSH-divergence (§2d) is RESOLVED. CR-13 (`"queued"` ConversationStatus) is RESOLVED. Backend shipped CR-10 (workspace id on `conversation.open` / `conversation.statusChanged`), CR-11 @@ -844,6 +850,183 @@ provider calls; the agent/model prompt never sees it (does not affect prompt cac point at `../dispatch-backend/...` (kept canonical — no worktree hack committed). An UNTRACKED symlink `dispatch-backend → backend` was created in the worktree parent, then `bun install` re-synced `node_modules/@dispatch/*` to pick up `[email protected]`. +## 2j. Vision & vision handoff → **CONSUMED ✅ (backend shipped; FE built + verified)** + +The backend shipped image/vision support: a user can attach images to a chat message, vision-capable +models receive them natively, and non-vision models get an auto-transcribed text description (the +"vision handoff"). The FE now pastes/picks/drops images in the composer, renders `image` chunks in the +transcript, and shows a vision badge in the model picker. Additive to `[email protected]` / +`[email protected]` (**NO version bump** — `ImageChunk`/`ImageInput` were added to the existing +versions; the FE's `file:` dep picks them up automatically, no re-pin needed). + +**New wire/transport types consumed + re-mirrored:** +- `ImageChunk` (`{ type: "image", url, mimeType? }`) — a NEW `Chunk` variant. `url` is a base64 data URL + (`data:image/…;base64,…`) OR an `http(s)://` URL. `ImageInput` (`{ url, mimeType? }`) is the transport- + facing input shape (`ChatRequest.images`); the orchestrator converts each into an `ImageChunk` on the + persisted user message. +- `ChatRequest.images?: readonly ImageInput[]` (so `ChatSendMessage` — which `extends ChatRequest` — + carries it on `chat.send`). `ChatQueueMessage` (steering) does NOT carry `images` — steering is + text-only (correctly: a mid-turn injection has no image surface). +- `ModelMetadata.vision?: boolean` — `true` when the model natively accepts images; absent/`false` → the + server's vision handoff transcribes images to text before the model sees them. +- Re-mirrored `.dispatch/wire.reference.md` (added `ImageChunk` to the `Chunk` union + the `ImageChunk`/ + `ImageInput` interfaces) and `.dispatch/transport-contract.reference.md` (added `images` to `ChatRequest`, + `vision` to `ModelMetadata`, + `ImageChunk`/`ImageInput`/`Computer`/`ComputerEntry` to the re-export). + +**FE (DONE + verified):** +- **Core (`core/chunks`):** the `assertChunkExhaustive` conformance guard caught the new `image` variant + (its purpose) → added the `case "image"` (this was the build break). `appendUserMessage(state, text, + images?)` now echoes a `[text, image, image, …]` user run (text first, then images in order; images-only + when text is empty). The `user-message` event carries ONLY text (never images — images arrive via history/ + loadSince + the optimistic echo), so its de-dup was generalized: it now scans the trailing provisional + USER run for a matching text chunk (not just the last chunk — which would be an image when images were + pasted, causing a duplicate text bubble). `applyHistory`'s during-generation de-dup was generalized to + match a multi-chunk user echo against the trailing committed user run by content equality + (`chunkContentEquals` — text/thinking/image/error/system/tool-call/tool-result), dropping the whole echo + only when fully backed (a partial match is kept until turn-seal drops all provisional wholesale). New + pure helpers `chunkContentEquals` + `trailingRun` (both internal). +16 reducer tests. +- **Transcript (`ChatView.svelte`):** a user `image` chunk renders as an `<img>` (lazy + async-decoded, + max-h-80) inside the user bubble, using the chunk's `url` directly. A non-vision model's persisted user + message keeps the original `image` chunk AND a `text` transcription (`[Image analysis (via <model>)]: …`) + in the SAME message — both render (image, then analysis text). The `read_image` tool call/result + renders like any other tool (its `toolName` is generic — no special-casing). +3 ChatView tests. +- **Composer (`Composer.svelte`):** image paste (clipboard `paste` — extracts image `File` items, + `preventDefault` only when an image is present so text paste still works), an attach-image button + + hidden `<input type=file accept=image/* multiple>`, and drag-drop onto the form. Files are read to + base64 data URLs (`FileReader.readAsDataURL`), capped at 8 MiB, staged as thumbnail previews with + remove buttons. `onSend` signature widened to `(text, images?)`; an image-only send (empty text) is + allowed; `images` is OMITTED on the wire (not `[]`) when none are staged (backward compatible). Steering + (`onQueue`) never forwards images. +6 Composer tests. +- **Store wiring:** `ChatStore.send(text, images?)` forwards `images` on the `chat.send` WS op + echoes + them; `AppStore.send(text, images?)` threads images through the draft→tab promotion; `App.svelte`'s + `handleSend(text, images?)` passes them through. The model catalog already captured `modelInfo` (now + with `vision`); `GET /models` is unchanged. +5 store/app tests. +- **Model picker (`ModelSelector.svelte` + `model-select.ts`):** new pure `isVisionModel(modelInfo, + fullName)`; the model dropdown marks vision-capable models (`" · vision"`), and an indicator below shows + "Vision — this model sees images natively" vs the handoff hint "Pasted images are auto-described". Wired + `modelInfo={store.modelInfo}` from `App.svelte`. +8 model-select/ModelSelector tests. + +**Invariants held:** +- `chat.send` STILL omits `cwd` (the persisted cwd wins) — only `images` was added to the message. +- The `user-message` event still carries only text; images are NEVER expected on it (a watcher fetches + them from history). The de-dup was made robust to the multi-chunk echo rather than reaching for images + on the event. +- `providerRetry`/`generating` unchanged; `image` is a normal committed/provisional chunk (it IS in the + `Chunk.type` union, unlike the transient `provider-retry`), so it persists + replays on reload. +- The `read_image` tool is rendered generically (no surface-id special-casing — the tool-name dispatch is + already identity-free). + +**Verification:** `svelte-check` 0/0; vitest **901/901** (run TWICE — no cross-test pollution; +34 new: +16 reducer, 3 ChatView, 6 Composer, 5 store/app, 4 model-select), biome clean, `vite build` succeeds (the +one CSS warning is PRE-EXISTING — `[file:path]`/`[heartbeat:elapsed]` attribute selectors, unrelated). +**Live probe NOT run:** the backend was not reachable headless at verify time (it is the user's process; +never booted headless). The full data path (paste → data URL → `chat.send` `images` → reducer echo → +transcript render; `GET /models` `vision` → badge; history `image` chunk → render) is covered by unit + +component + store tests. To confirm end-to-end: start the backend, paste an image into the composer with +a vision model selected (e.g. any `kimi/*`), send, confirm the image renders + the model responds to it; +then switch to a non-vision model (e.g. `umans/glm-5.2`), paste an image, send, confirm the image renders +AND a numbered placeholder `[Image N attached — call consult_vision…]` text bubble appears (the non-vision +handoff — see the update below; the old `[Image analysis (via …)]` auto-transcription is GONE); confirm +the vision badge shows/hides per model in the picker. + +### 2j-update. consult_vision tool + vision settings API → **CONSUMED ✅ (backend updated; FE built + verified)** + +A follow-up to §2j. The backend revised the vision surface: the `read_image` tool is REPLACED by +`consult_vision`, non-vision models get NUMBERED PLACEHOLDERS (not auto-transcriptions), and there is a +NEW global vision-settings API. Additive to `[email protected]` / `[email protected]` (NO version bump); +re-mirrored `.dispatch/transport-contract.reference.md` (added `VisionSettingsResponse` / +`SetVisionSettingsRequest` + a delta note). + +**What changed (backend):** +- **`read_image` → `consult_vision`** (`{ question: string (req), imageIds?: number[], path?: string }`): + opens a NEW conversation tab with a vision-capable model (Kimi), attaches the image + question, returns + the conversation id + the vision model's answer (suggests the dispatch CLI for follow-ups). Rendered like + any tool call/result (generic `toolName` dispatch — no special-casing). +- **Non-vision models get NUMBERED PLACEHOLDERS** instead of auto-transcriptions: a pasted image on a + non-vision model persists an `image` chunk + a `text` chunk `[Image N attached — call consult_vision + with imageIds=[N] and a specific question to analyze it]`. (The old `[Image analysis (via <model>)]: …` + auto-transcription is GONE.) +- **Image compaction** (transparent): when a vision model has > `imageLimit` images in history, the oldest + are transcribed to `[Compacted image]: <description>` `text` chunks (the persisted `image` chunk stays + for rendering). Both placeholder + compacted chunks are REGULAR `text` chunks — render as-is (no special + handling). +- **NEW global vision settings API:** `GET /settings/vision` → `VisionSettingsResponse` + (`{ imageLimit: number (default 10), compactionModel: string|null (null = auto) }`); `PUT /settings/vision` + ← `SetVisionSettingsRequest` (partial: `imageLimit?` non-negative int, 0 = disable compaction; + `compactionModel?` `<key>/<model>` or null). + +**FE (DONE + verified):** +- **Tool rendering:** the ChatView `read_image` test → `consult_vision` (rendering is generic — renders by + `toolName`, so no component change; only the test name/input updated). +2 ChatView tests for the + placeholder text chunk + the compacted-image text chunk (both render as regular text). +- **New `vision` feature library** (`src/features/vision/`): pure `logic/view-model.ts` (32 tests) — + `VisionSettings`/`VisionSettingsPatch` (owned locally, consumer-defines-port; the shapes are a plain REST + surface, mirroring heartbeat/mcp), `LoadVisionSettings`/`SaveVisionSettings` ports + result types, + `normalizeVisionSettings` (network-seam coercion — a malformed body can't crash the renderer), + `parseImageLimit`/`imageLimitChanged` (dirty-check), `compactionModelOptions` (filters `GET /models` to + vision-capable models via the chat feature's public `isVisionModel` export + an "Auto" sentinel), + `selectedCompactionValue`/`compactionModelFromValue` round-trip, `imageLimitLabel`; `ui/VisionSettingsView.svelte` + (imageLimit text input + Save, compactionModel dropdown with Auto + vision-capable models, load-on-mount, + save-on-change, error/saved feedback; 9 component tests); `index.ts`. +- **Cross-unit seam:** `isVisionModel` was added to `features/chat`'s public `index.ts` (additive — the + vision feature imports it through the public surface, not the chat internals). +- **Store wiring** (`src/app/store.svelte.ts`): `visionSettings` reactive state + `refreshVisionSettings()` + (`GET /settings/vision`, normalized at the seam) + `setVisionSettings(patch)` (`PUT /settings/vision`, + returns merged settings) + `VisionSettingsResult`; seeded on boot; exposed on `AppStore`. +4 store tests. +- **Mounted in `App.svelte`:** a new "Vision" sidebar view kind (`viewKinds`) + `VisionSettingsView` in the + `viewContent` snippet (not conversation-scoped — no `{#key}`); `loadVisionSettings`/`saveVisionSettings` + adapters wrap the store; `visionManifest` in `loadedModules`. + +**Verification:** `svelte-check` 0/0; vitest **948/948** (run TWICE — no cross-test pollution; +47 new +since the §2j baseline: 32 vision view-model, 9 VisionSettingsView, 4 store, +2 ChatView +placeholder/compacted, and the read_image→consult_vision test update), biome clean, `vite build` succeeds. +**Live probe NOT run** (backend not reachable headless). To confirm end-to-end: open the Vision sidebar +view, confirm the imageLimit + compactionModel load; change imageLimit → Save → confirm it persists on +reload; set compactionModel to a vision model → confirm the dropdown reflects it; paste an image with a +non-vision model → confirm the `[Image N attached — call consult_vision…]` placeholder renders (not an +auto-transcription); trigger a `consult_vision` tool call → confirm it renders like a tool. + +### 2j-update-2. Image storage (tmp, not SQLite) → **CONSUMED ✅ (backend updated; FE built + verified)** + +A follow-up to §2j. The backend no longer persists images as base64 data URLs in the conversation store — +they are saved to a tmp directory and served via HTTP. The `ImageChunk.url` field's FORMAT changed (the +TYPE is unchanged — still `string`); `GET /images/:conversationId/:imageId` is a new endpoint serving raw +bytes + the correct Content-Type. **NO wire/transport-contract type change** (behavior only); re-mirrored +the delta notes in `.dispatch/wire.reference.md` + `.dispatch/transport-contract.reference.md`. + +**What changed (backend):** +- **BEFORE:** `ImageChunk.url` was a base64 data URL (`data:image/png;base64,…`). +- **NOW:** `ImageChunk.url` for PERSISTED chunks (history/replay) is a compact relative HTTP path + (`/images/<conversationId>/<uuid>.png`), served by `GET /images/:conversationId/:imageId` (raw image + bytes + Content-Type). Images live on disk under tmp, NOT in the SQLite conversation store (keeps + payloads small). +- **`ChatRequest.images` (`ImageInput.url`) is UNCHANGED** — the FE still sends data URLs; the backend + saves them to tmp and returns compact paths in the persisted chunks. +- **Optimistic echo:** the FE's provisional echo still uses the data URL it sent (immediate render); + when the persisted chunk arrives (via `loadSince`/`syncTail`/event stream), it carries the compact path + and the FE switches to rendering via the HTTP endpoint. +- The vision settings API, `consult_vision`, and image compaction are UNCHANGED (compaction resolves + compact URLs internally). + +**FE (DONE + verified):** +- **New pure helper `resolveImageUrl(url, apiBase)`** (`core/chunks/image-url.ts`, +8 tests, exported from + `core/chunks` + re-exported from `features/chat`): a `data:` URL or an `http(s)://` URL passes through + unchanged; a relative path (`/images/…`) is prepended with the API base (no double slash; an empty base + yields a root-relative path a browser resolves against its origin). Pure — zero DOM/Svelte. +- **`ChatView.svelte`:** new `apiBaseUrl` prop (default `""`); the `<img src>` now uses + `resolveImageUrl(rendered.chunk.url, apiBaseUrl)`. The optimistic echo's data URL + any absolute URL + pass through; persisted relative paths resolve against the base. +3 ChatView tests (relative-path + resolution, data-URL pass-through with a base set, root-relative when no base). +- **Store + wiring:** `httpBase` (the resolved HTTP API base) is now exposed as a getter on `AppStore`; + `App.svelte` passes `apiBaseUrl={store.httpBase}` to `ChatView` AND to the heartbeat `RunModal` (its + `ChatView` also renders image chunks — added an `apiBaseUrl` prop there, threaded from `App.svelte`). + +**Verification:** `svelte-check` 0/0; vitest **959/959** (run TWICE — no cross-test pollution; +11 since the +prior commit: 8 `resolveImageUrl`, 3 ChatView resolution), biome clean, `vite build` succeeds. +**Live probe NOT run** (backend not reachable headless). To confirm end-to-end: paste an image with a +vision model, send, confirm the image renders immediately (data-URL echo) AND continues to render after +the turn seals (the persisted compact-path `/images/…` resolved against `httpBase`); reload the +conversation → confirm the persisted image renders from the `/images/…` endpoint (not a data URL). --- diff --git a/src/app/App.svelte b/src/app/App.svelte index b46885c..59f949c 100644 --- a/src/app/App.svelte +++ b/src/app/App.svelte @@ -1,5 +1,5 @@ <script lang="ts"> - import type { ReasoningEffort } from "@dispatch/transport-contract"; + import type { ImageInput, ReasoningEffort } from "@dispatch/transport-contract"; import type { InvokeMessage } from "@dispatch/ui-contract"; import { tick } from "svelte"; import Table from "../components/Table.svelte"; @@ -86,6 +86,13 @@ type SaveSystemPrompt as SaveSystemPromptAlias, manifest as systemPromptManifest, } from "../features/system-prompt"; + import { + VisionSettingsView, + manifest as visionManifest, + type LoadVisionSettingsResult, + type SaveVisionSettingsResult, + type VisionSettingsPatch, + } from "../features/vision"; import type { AppStore } from "./store.svelte"; import ErrorModal from "./ErrorModal.svelte"; import { createLocalStore } from "../adapters/local-storage"; @@ -156,6 +163,7 @@ systemPromptManifest, heartbeatManifest, concurrencyManifest, + visionManifest, ].map((m) => [m.name, m.description] as const); // Smart-scroll: keep the transcript pinned to the bottom while it streams, @@ -279,8 +287,8 @@ store.invoke(msg.surfaceId, msg.actionId, msg.payload); } - function handleSend(text: string) { - store.send(text); + function handleSend(text: string, images?: readonly ImageInput[]): void { + store.send(text, images); } function handleQueue(text: string) { @@ -342,6 +350,27 @@ : { ok: false, error: result.error }; } + // Adapt the store's global vision-settings API to the vision feature's ports. + async function loadVisionSettings(): Promise<LoadVisionSettingsResult> { + // The store seeds `visionSettings` on boot; a refresh keeps it current. + await store.refreshVisionSettings(); + const settings = store.visionSettings; + if (settings === null) { + return { ok: false, error: "Vision settings not available." }; + } + return { ok: true, settings }; + } + + async function saveVisionSettings( + patch: VisionSettingsPatch, + ): Promise<SaveVisionSettingsResult> { + const result = await store.setVisionSettings(patch); + if (result === null) return { ok: false, error: "Vision settings not available." }; + return result.ok + ? { ok: true, settings: result.settings } + : { ok: false, error: result.error }; + } + // Adapt the store's chat-limit result to the settings feature's port. On a // raise the active chat refills (prepends older history); preserve the // reader's viewport over the prepend (the manual analogue of CSS scroll @@ -572,6 +601,7 @@ onShowEarlier={handleShowEarlier} thinkingKeyBase={store.activeChat.thinkingKeyBase} providerRetry={store.activeChat.providerRetry} + apiBaseUrl={store.httpBase} /> {/key} </div> @@ -662,6 +692,7 @@ closeChat={closeRunChat} stopRun={stopHeartbeatRun} onClose={() => (heartbeatRun = null)} + apiBaseUrl={store.httpBase} /> {/key} {/if} @@ -684,7 +715,12 @@ {/key} {:else if kind === "model"} <div class="flex flex-col gap-3"> - <ModelSelector models={store.models} selected={store.activeModel} onSelect={handleSelectModel} /> + <ModelSelector + models={store.models} + selected={store.activeModel} + onSelect={handleSelectModel} + modelInfo={store.modelInfo} + /> <!-- Keyed on the workspace conversation (active tab OR draft) so the inputs re-mount per conversation — incl. switching between drafts — and can't bleed across tabs. Editable for a draft too (cwd + effort apply from turn 1). --> @@ -743,7 +779,9 @@ {/if} {/key} {:else if kind === "compaction"} - <!-- Re-mount per conversation so the percent + feedback can't bleed across tabs. --> + <!-- Message compaction is per-conversation (keyed so the percent + feedback + can't bleed across tabs). Vision (image-compaction) settings are GLOBAL, + so they stay mounted across conversation switches (no {#key}). --> {#key store.currentConversationId} <CompactionView percent={store.compactPercent} @@ -752,6 +790,13 @@ savePercent={saveCompactPercent} /> {/key} + <div class="divider my-1 text-xs text-base-content/40">Image compaction</div> + <VisionSettingsView + models={store.models} + modelInfo={store.modelInfo} + load={loadVisionSettings} + save={saveVisionSettings} + /> {:else if kind === "system-prompt"} <!-- Global system prompt template. Opens a full-page modal editor (half template / half variable palette). Not conversation-scoped (no {#key}). --> diff --git a/src/app/store.svelte.ts b/src/app/store.svelte.ts index 0506f91..ead27a6 100644 --- a/src/app/store.svelte.ts +++ b/src/app/store.svelte.ts @@ -27,6 +27,7 @@ import type { SetReasoningEffortRequest, SetSystemPromptTemplateRequest, SetTitleRequest, + SetVisionSettingsRequest, SystemPromptTemplateResponse, SystemPromptVariable, SystemPromptVariablesResponse, @@ -35,7 +36,7 @@ import type { WarmResponse, } from "@dispatch/transport-contract"; import type { SubscribeMessage, SurfaceServerMessage, SurfaceSpec } from "@dispatch/ui-contract"; -import type { ComputerEntry, ConversationStatus } from "@dispatch/wire"; +import type { ComputerEntry, ConversationStatus, ImageInput } from "@dispatch/wire"; import { untrack } from "svelte"; import { createIdbChunkStore } from "../adapters/idb"; import { createLocalStore } from "../adapters/local-storage"; @@ -78,6 +79,11 @@ import type { import { normalizeHeartbeatConfig, normalizeHeartbeatRuns } from "../features/heartbeat"; import type { Tab, TabsState } from "../features/tabs"; import { createTabsStore, deriveTitle, type TabsStore } from "../features/tabs"; +import { + normalizeVisionSettings, + type VisionSettings, + type VisionSettingsPatch, +} from "../features/vision"; import { resolveHttpUrl } from "./resolve-http-url"; import { resolveWsUrl } from "./resolve-ws-url"; import { randomId } from "./uuid"; @@ -134,6 +140,11 @@ export type CompactPercentResult = | { readonly ok: true; readonly percent: number } | { readonly ok: false; readonly error: string }; +/** Outcome of `PUT /settings/vision` (global vision-settings save). */ +export type VisionSettingsResult = + | { readonly ok: true; readonly settings: VisionSettings } + | { readonly ok: false; readonly error: string }; + /** Outcome of `GET /system-prompt` (global template load). */ export type SystemPromptLoadResult = | { readonly ok: true; readonly template: string } @@ -157,6 +168,12 @@ export interface AppStore { readonly activeConversationId: string | null; /** The workspace currently in view (URL slug); tabs are filtered to it. */ readonly activeWorkspaceId: string; + /** + * The resolved HTTP API base URL (e.g. `http://localhost:24203`). Used to + * resolve relative image URLs served by the backend (`/images/…`) into + * absolute URLs for `<img src>`. + */ + readonly httpBase: string; readonly activeChat: ChatStore; readonly models: readonly string[]; /** Per-model metadata (contextWindow, etc.) from `GET /models`. */ @@ -171,7 +188,14 @@ export interface AppStore { readonly storage: Storage | undefined; /** The current spec for one surface by id (discovery-by-id), or null if absent. */ surface(surfaceId: string): SurfaceSpec | null; - send(text: string): void; + /** + * Send a user message (start a turn). Forwards any staged `images` + * (`ImageInput[]` — base64 data URLs / https URLs) on the `chat.send` op; + * the server passes them to a vision-capable model natively or transcribes + * them via vision handoff for a non-vision model. Omitted on the wire when + * none are staged. On a draft, promotes to a tab first. + */ + send(text: string, images?: readonly ImageInput[]): void; /** * Enqueue a steering message onto the focused conversation's queue * (`chat.queue` WS op). While a turn is generating, the message is delivered @@ -273,6 +297,23 @@ export interface AppStore { */ setCompactPercent(percent: number): Promise<CompactPercentResult | null>; /** + * The GLOBAL vision settings (`GET /settings/vision`): `imageLimit` (max + * native images per turn before compaction; 0 = disabled) + `compactionModel` + * (which vision model transcribes old images; null = auto). Shared across all + * conversations. Seeded on boot; `null` = not yet fetched. + */ + readonly visionSettings: VisionSettings | null; + /** + * Refetch the global vision settings (`GET /settings/vision`). Called by the + * vision-settings view on mount; also seeded on boot. + */ + refreshVisionSettings(): Promise<void>; + /** + * Save a PARTIAL vision-settings update (`PUT /settings/vision`). Either + * field may be omitted. Returns the merged settings on success. + */ + setVisionSettings(patch: VisionSettingsPatch): Promise<VisionSettingsResult | null>; + /** * Fetch the workspace conversation's language-server status (`GET /conversations/:id/lsp`). * The backend lazily spawns servers, so this may take a moment on the first call for a cwd. */ @@ -662,6 +703,22 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { // backend on focus change; null = not yet fetched. 0 = disabled. let compactPercent = $state<number | null>(null); + // The GLOBAL vision settings (shared across all conversations). Seeded on + // boot; null = not yet fetched. + let visionSettings = $state<VisionSettings | null>(null); + + /** Refetch the global vision settings (`GET /settings/vision`). */ + async function refreshVisionSettings(): Promise<void> { + try { + const res = await fetchImpl(`${httpBase}/settings/vision`); + if (!res.ok) return; + const data = normalizeVisionSettings(await res.json()); + visionSettings = data; + } catch (err) { + reportError("Failed to load vision settings", err); + } + } + /** Refetch the workspace conversation's compact percent (works for a draft too). */ async function refreshCompactPercent(): Promise<void> { const id = workspaceConversationId(); @@ -1106,6 +1163,7 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { void refreshModel(); void refreshReasoningEffort(); void refreshCompactPercent(); + void refreshVisionSettings(); // Fetch the authoritative open-conversation list from the backend (cross- // device tab sync). Merges with the localStorage-restored tabs: opens new @@ -1122,6 +1180,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { get activeWorkspaceId(): string { return activeWorkspaceId; }, + get httpBase(): string { + return httpBase; + }, setActiveWorkspace(workspaceId: string): void { activeWorkspaceId = workspaceId; // Reset to a fresh draft scoped to the new workspace so a new chat is @@ -1182,6 +1243,12 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { get compactPercent(): number | null { return compactPercent; }, + get visionSettings(): VisionSettings | null { + return visionSettings; + }, + async refreshVisionSettings(): Promise<void> { + await refreshVisionSettings(); + }, get chatLimit(): number { return chatLimit; }, @@ -1196,7 +1263,7 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { return getSurfaceSpec(protocol, surfaceId); }, - send(text: string): void { + send(text: string, images?: readonly ImageInput[]): void { if (tabsStore.activeConversationId === null) { // Draft: promote to tab on first send const conversationId = draftConversationId; @@ -1224,9 +1291,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { void refreshReasoningEffort(); void refreshCompactPercent(); // Now send on the promoted store - chatStores.get(conversationId)?.send(text); + chatStores.get(conversationId)?.send(text, images); } else { - activeChat.send(text); + activeChat.send(text, images); } }, @@ -1540,6 +1607,32 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { } }, + async setVisionSettings(patch: VisionSettingsPatch): Promise<VisionSettingsResult | null> { + const body: SetVisionSettingsRequest = patch; + try { + const res = await fetchImpl(`${httpBase}/settings/vision`, { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) { + const errBody = (await res.json().catch(() => null)) as { error?: string } | null; + return { + ok: false, + error: errBody?.error ?? `Set vision settings failed (HTTP ${res.status})`, + }; + } + const data = normalizeVisionSettings(await res.json()); + visionSettings = data; + return { ok: true, settings: data }; + } catch (err) { + return { + ok: false, + error: err instanceof Error ? err.message : "Set vision settings request failed", + }; + } + }, + async setChatLimit(limit: number): Promise<ChatLimitResult> { const next = normalizeChatLimit(limit); chatLimitStore.save(next); diff --git a/src/app/store.test.ts b/src/app/store.test.ts index b47a378..2cf473c 100644 --- a/src/app/store.test.ts +++ b/src/app/store.test.ts @@ -423,6 +423,34 @@ describe("createAppStore", () => { store.dispose(); }); + it("sending from draft forwards staged images on chat.send", () => { + const ws = fakeSocket(); + const store = createAppStore({ + socketFactory: () => ws, + fetchImpl: fakeFetchImpl(), + localStorage: createFakeStorage(), + }); + ws.resolveOpen(); + ws.sent.length = 0; + + const images = [ + { url: "data:image/png;base64,AAAA", mimeType: "image/png" }, + { url: "https://example.com/x.jpg" }, + ]; + store.send("describe these", images); + + const msgs = parseSent(ws); + const chatSend = msgs.find((m) => (m as { type: string }).type === "chat.send") as + | { type: string; message: string; images?: { url: string }[] } + | undefined; + expect(chatSend).toBeTruthy(); + expect(chatSend?.images).toEqual(images); + // The optimistic echo includes the image chunks. + expect(store.activeChat.chunks.some((c) => c.chunk.type === "image")).toBe(true); + + store.dispose(); + }); + it("an incoming chat.delta renders in the transcript", () => { const ws = fakeSocket(); const store = createAppStore({ @@ -1703,3 +1731,127 @@ describe("createAppStore", () => { store.dispose(); }); }); + +describe("createAppStore — vision settings (global)", () => { + function visionFetch(initial: { imageLimit: number; compactionModel: string | null }): { + fetchImpl: typeof fetch; + puts: { imageLimit?: number; compactionModel?: string | null }[]; + } { + let current = initial; + const puts: { imageLimit?: number; compactionModel?: string | null }[] = []; + return { + puts, + fetchImpl: async (input: string | URL | Request, init?: RequestInit): Promise<Response> => { + const url = + typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.endsWith("/models")) { + return new Response( + JSON.stringify({ + models: ["kimi/k2", "umans/glm-5.2"], + modelInfo: { "kimi/k2": { vision: true } }, + }), + { status: 200 }, + ); + } + if (url.endsWith("/settings/vision")) { + if (init?.method === "PUT") { + const text = typeof init.body === "string" ? init.body : ""; + const body = text ? (JSON.parse(text) as object) : {}; + puts.push(body as { imageLimit?: number; compactionModel?: string | null }); + current = { ...current, ...(body as object) } as { + imageLimit: number; + compactionModel: string | null; + }; + } + return new Response(JSON.stringify(current), { status: 200 }); + } + // Default: empty history + no cwd for the other endpoints. + return new Response(JSON.stringify({ chunks: [], latestSeq: 0 }), { status: 200 }); + }, + }; + } + + it("loads vision settings on boot (GET /settings/vision)", async () => { + const { fetchImpl } = visionFetch({ imageLimit: 7, compactionModel: "kimi/k2" }); + const store = createAppStore({ + socketFactory: () => fakeSocket(), + fetchImpl, + localStorage: createFakeStorage(), + }); + fakeSocket().resolveOpen(); // not strictly needed for HTTP + + await vi.waitFor(() => { + expect(store.visionSettings).toEqual({ imageLimit: 7, compactionModel: "kimi/k2" }); + }); + store.dispose(); + }); + + it("setVisionSettings PUTs a partial update and reflects the merged settings", async () => { + const ctx = visionFetch({ imageLimit: 10, compactionModel: null }); + const store = createAppStore({ + socketFactory: () => fakeSocket(), + fetchImpl: ctx.fetchImpl, + localStorage: createFakeStorage(), + }); + + await vi.waitFor(() => { + expect(store.visionSettings?.imageLimit).toBe(10); + }); + + const result = await store.setVisionSettings({ imageLimit: 3 }); + expect(result?.ok).toBe(true); + if (result?.ok) { + expect(result.settings.imageLimit).toBe(3); + expect(result.settings.compactionModel).toBeNull(); + } + expect(ctx.puts).toEqual([{ imageLimit: 3 }]); + expect(store.visionSettings?.imageLimit).toBe(3); + + // A second save updates compactionModel only. + const result2 = await store.setVisionSettings({ compactionModel: "kimi/k2" }); + expect(result2?.ok).toBe(true); + expect(ctx.puts).toEqual([{ imageLimit: 3 }, { compactionModel: "kimi/k2" }]); + expect(store.visionSettings?.compactionModel).toBe("kimi/k2"); + store.dispose(); + }); + + it("refreshVisionSettings refetches (load adapter)", async () => { + const { fetchImpl } = visionFetch({ imageLimit: 5, compactionModel: null }); + const store = createAppStore({ + socketFactory: () => fakeSocket(), + fetchImpl, + localStorage: createFakeStorage(), + }); + + await store.refreshVisionSettings(); + expect(store.visionSettings).toEqual({ imageLimit: 5, compactionModel: null }); + store.dispose(); + }); + + it("surfaces a PUT error", async () => { + const fetchImpl: typeof fetch = async (input, init) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.endsWith("/settings/vision")) { + if (init?.method === "PUT") { + return new Response(JSON.stringify({ error: "invalid imageLimit" }), { status: 400 }); + } + return new Response(JSON.stringify({ imageLimit: 10, compactionModel: null }), { + status: 200, + }); + } + return new Response(JSON.stringify({ models: [] }), { status: 200 }); + }; + const store = createAppStore({ + socketFactory: () => fakeSocket(), + fetchImpl, + localStorage: createFakeStorage(), + }); + + const result = await store.setVisionSettings({ imageLimit: -1 }); + expect(result?.ok).toBe(false); + if (result !== null && !result.ok) { + expect(result.error).toContain("invalid imageLimit"); + } + store.dispose(); + }); +}); diff --git a/src/core/chunks/image-url.test.ts b/src/core/chunks/image-url.test.ts new file mode 100644 index 0000000..8a79c09 --- /dev/null +++ b/src/core/chunks/image-url.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { resolveImageUrl } from "./image-url"; + +const BASE = "http://localhost:24203"; + +describe("resolveImageUrl", () => { + it("returns a data URL as-is (the optimistic echo / a pasted image)", () => { + const dataUrl = "data:image/png;base64,iVBORw0KGgo="; + expect(resolveImageUrl(dataUrl, BASE)).toBe(dataUrl); + }); + + it("returns an absolute http URL as-is", () => { + const abs = "https://example.com/img.png"; + expect(resolveImageUrl(abs, BASE)).toBe(abs); + }); + + it("prepends the api base to a relative /images/ path", () => { + expect(resolveImageUrl("/images/conv-123/abc-456.png", BASE)).toBe( + "http://localhost:24203/images/conv-123/abc-456.png", + ); + }); + + it("does not double the slash when the base has a trailing slash", () => { + expect(resolveImageUrl("/images/c/x.png", "http://localhost:24203/")).toBe( + "http://localhost:24203/images/c/x.png", + ); + }); + + it("adds a leading slash to a path-relative url without one", () => { + expect(resolveImageUrl("images/c/x.png", BASE)).toBe("http://localhost:24203/images/c/x.png"); + }); + + it("returns the relative path as-is when apiBase is empty (root-relative)", () => { + // A browser resolves a root-relative `/images/…` against the document origin. + expect(resolveImageUrl("/images/c/x.png", "")).toBe("/images/c/x.png"); + }); + + it("handles a relative path with an empty apiBase (path-relative without slash)", () => { + expect(resolveImageUrl("images/c/x.png", "")).toBe("/images/c/x.png"); + }); + + it("returns a data URL as-is even with an empty apiBase", () => { + const dataUrl = "data:image/jpeg;base64,AAAA"; + expect(resolveImageUrl(dataUrl, "")).toBe(dataUrl); + }); +}); diff --git a/src/core/chunks/image-url.ts b/src/core/chunks/image-url.ts new file mode 100644 index 0000000..e5ec756 --- /dev/null +++ b/src/core/chunks/image-url.ts @@ -0,0 +1,35 @@ +/** + * Resolve an `ImageChunk.url` into a renderable `<img src>` value. + * + * Persisted image chunks now carry a COMPACT HTTP path + * (`/images/<conversationId>/<uuid>.png`) served by the backend — NOT a base64 + * data URL (images are stored on disk under tmp, not in the conversation store, + * to keep SQLite payloads small). The optimistic echo (what the FE just sent in + * `ChatRequest.images`) still carries a data URL, and a chunk could also carry + * an absolute `http(s)://` URL, so the resolution is format-aware: + * + * - `data:` URL → returned as-is (the optimistic echo / a pasted data URL). + * - `http(s)://` → returned as-is (an absolute URL already). + * - anything else (a relative path like `/images/…`) → `apiBase` is prepended + * (with no double slash). An empty `apiBase` leaves a root-relative path, + * which a browser resolves against the document origin. + * + * Pure: input → output, zero DOM, zero Svelte. + * + * @param url The chunk's `url` (data URL, absolute, or relative path). + * @param apiBase The HTTP API base URL (e.g. `http://localhost:24203`). + */ +export function resolveImageUrl(url: string, apiBase: string): string { + if (url.startsWith("data:") || url.startsWith("http://") || url.startsWith("https://")) { + return url; + } + // A relative path (e.g. `/images/…`) — normalize to a leading slash and + // prepend the api base. With an empty base this yields a root-relative path + // (a browser resolves `/images/…` against the document origin). + const path = url.startsWith("/") ? url : `/${url}`; + if (apiBase.length === 0) return path; + // Join without a double slash: strip a trailing slash from the base, then + // append the (leading-slash) path verbatim. + const base = apiBase.endsWith("/") ? apiBase.slice(0, -1) : apiBase; + return `${base}${path}`; +} diff --git a/src/core/chunks/index.ts b/src/core/chunks/index.ts index eea2303..bdd6ce4 100644 --- a/src/core/chunks/index.ts +++ b/src/core/chunks/index.ts @@ -1,5 +1,6 @@ export type { RenderGroup, ToolBatchEntry } from "./groups"; export { groupRenderedChunks } from "./groups"; +export { resolveImageUrl } from "./image-url"; export { appendUserMessage, applyHistory, diff --git a/src/core/chunks/reducer.test.ts b/src/core/chunks/reducer.test.ts index 8a2e1b7..058552e 100644 --- a/src/core/chunks/reducer.test.ts +++ b/src/core/chunks/reducer.test.ts @@ -1,4 +1,5 @@ import type { + ImageInput, StepId, StoredChunk, TurnDoneEvent, @@ -869,3 +870,155 @@ describe("appendUserMessage", () => { expect(s.provisional[1]?.chunk).toEqual({ type: "text", text: "user msg" }); }); }); + +const PNG = "data:image/png;base64,AAAA"; +const JPG = "data:image/jpeg;base64,BBBB"; + +describe("appendUserMessage — images (vision handoff)", () => { + it("echoes text then image chunks in order", () => { + const images: ImageInput[] = [ + { url: PNG, mimeType: "image/png" }, + { url: JPG, mimeType: "image/jpeg" }, + ]; + let s = initialState(); + s = appendUserMessage(s, "what's this?", images); + const chunks = selectChunks(s); + expect(chunks).toHaveLength(3); + expect(chunks[0]?.role).toBe("user"); + expect(chunks[0]?.chunk).toEqual({ type: "text", text: "what's this?" }); + expect(chunks[1]?.chunk).toEqual({ type: "image", url: PNG, mimeType: "image/png" }); + expect(chunks[2]?.chunk).toEqual({ type: "image", url: JPG, mimeType: "image/jpeg" }); + expect(chunks.every((c) => c.provisional && c.seq === null)).toBe(true); + }); + + it("omits mimeType when not provided", () => { + let s = initialState(); + s = appendUserMessage(s, "look", [{ url: PNG }]); + const img = selectChunks(s)[1]?.chunk; + expect(img).toEqual({ type: "image", url: PNG }); + expect(img).not.toHaveProperty("mimeType"); + }); + + it("echoes images-only when text is empty (no text chunk)", () => { + let s = initialState(); + s = appendUserMessage(s, "", [{ url: PNG, mimeType: "image/png" }]); + const chunks = selectChunks(s); + expect(chunks).toHaveLength(1); + expect(chunks[0]?.chunk.type).toBe("image"); + }); + + it("echoes nothing for empty text + empty images", () => { + let s = initialState(); + s = foldEvent(s, turnStart("t1")); + s = foldEvent(s, textDelta("t1", "partial")); + s = appendUserMessage(s, "", []); + // Defensive flush still happened; no user chunk added. + const users = selectChunks(s).filter((c) => c.role === "user"); + expect(users).toHaveLength(0); + expect(s.accumulating).toBeNull(); + }); + + it("skips images with an empty url", () => { + let s = initialState(); + s = appendUserMessage(s, "hi", [ + { url: "", mimeType: "image/png" }, + { url: PNG, mimeType: "image/png" }, + ]); + const chunks = selectChunks(s); + expect(chunks).toHaveLength(2); // text + the one valid image + expect(chunks[1]?.chunk.type).toBe("image"); + }); + + it("groups text + images into one user ChatMessage", () => { + let s = initialState(); + s = appendUserMessage(s, "see this", [{ url: PNG }]); + const msgs = selectMessages(s); + expect(msgs).toHaveLength(1); + expect(msgs[0]?.role).toBe("user"); + expect(msgs[0]?.chunks).toHaveLength(2); + }); +}); + +describe("foldEvent — user-message dedups against a text+image echo", () => { + const userMessage = (text: string): TurnInputEvent => ({ + type: "user-message", + conversationId: "c1", + turnId: "t1", + text, + }); + + it("does not duplicate the text when images follow it in the echo", () => { + let s = initialState(); + s = appendUserMessage(s, "hi", [{ url: PNG, mimeType: "image/png" }]); + expect(selectChunks(s)).toHaveLength(2); // text + image + s = foldEvent(s, userMessage("hi")); + const users = selectChunks(s).filter((c) => c.role === "user"); + expect(users).toHaveLength(2); // unchanged — no duplicate text + expect(users[0]?.chunk.type).toBe("text"); + expect(users[1]?.chunk.type).toBe("image"); + expect(s.generating).toBe(true); + }); + + it("still appends when the echoed text differs", () => { + let s = initialState(); + s = appendUserMessage(s, "first", [{ url: PNG }]); + s = foldEvent(s, userMessage("second")); + const users = selectChunks(s).filter((c) => c.role === "user"); + expect(users.filter((c) => c.chunk.type === "text")).toHaveLength(2); + }); +}); + +describe("applyHistory — multi-chunk image echo is superseded by committed", () => { + it("drops the provisional [text, image] echo when committed arrives during generation", () => { + let s = initialState(); + s = appendUserMessage(s, "hi", [{ url: PNG, mimeType: "image/png" }]); + s = foldEvent(s, turnStart("t1")); + expect(s.provisional).toHaveLength(2); + + s = applyHistory(s, [ + storedChunk(1, "user", { type: "text", text: "hi" }), + storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }), + ]); + expect(s.provisional).toEqual([]); + expect(s.committed).toHaveLength(2); + expect(s.committed[0]?.chunk).toEqual({ type: "text", text: "hi" }); + expect(s.committed[1]?.chunk).toEqual({ type: "image", url: PNG, mimeType: "image/png" }); + }); + + it("keeps the echo when committed is only a partial match (img not yet persisted)", () => { + let s = initialState(); + s = appendUserMessage(s, "hi", [ + { url: PNG, mimeType: "image/png" }, + { url: JPG, mimeType: "image/jpeg" }, + ]); + s = foldEvent(s, turnStart("t1")); + // Only the text + first image have been persisted so far. + s = applyHistory(s, [ + storedChunk(1, "user", { type: "text", text: "hi" }), + storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }), + ]); + // Echo is [text, img1, img2]; committed run is [text, img1] — not a full + // match (echo is longer) → keep the echo (turn-seal will drop it wholesale). + expect(s.provisional.length).toBeGreaterThan(0); + }); + + it("renders committed image chunks from history (a non-vision transcription turn)", () => { + // The server persists the original image chunk AND the transcription text + // in the SAME user message: render both (image, then analysis text). + let s = initialState(); + s = applyHistory(s, [ + storedChunk(1, "user", { type: "text", text: "describe this" }), + storedChunk(2, "user", { type: "image", url: PNG, mimeType: "image/png" }), + storedChunk(3, "user", { + type: "text", + text: "[Image analysis (via kimi/k2)]: a red square", + }), + storedChunk(4, "assistant", { type: "text", text: "the square is red" }), + ]); + const msgs = selectMessages(s); + expect(msgs).toHaveLength(2); // one user message (3 chunks) + one assistant + expect(msgs[0]?.role).toBe("user"); + expect(msgs[0]?.chunks).toHaveLength(3); + expect(msgs[0]?.chunks[1]).toEqual({ type: "image", url: PNG, mimeType: "image/png" }); + }); +}); diff --git a/src/core/chunks/reducer.ts b/src/core/chunks/reducer.ts index 2152de3..64e41b9 100644 --- a/src/core/chunks/reducer.ts +++ b/src/core/chunks/reducer.ts @@ -1,4 +1,5 @@ -import type { AgentEvent, Chunk, StoredChunk } from "@dispatch/wire"; +import type { AgentEvent, Chunk, ImageInput, Role, StoredChunk } from "@dispatch/wire"; +import { assertChunkExhaustive } from "../wire/conformance"; import type { AccumulatingChunk, ProvisionalChunk, TranscriptState } from "./types"; /** The initial empty transcript state. */ @@ -43,6 +44,63 @@ function flushAccumulating( } /** + * Content equality for two chunks of the SAME role (used to de-dup an + * optimistic echo against the authoritative committed version). Compares the + * discriminating payload only — `stepId` (generation provenance) is ignored + * since it is absent on provisional echoes but present on committed tool chunks. + */ +function chunkContentEquals(a: Chunk, b: Chunk): boolean { + if (a.type !== b.type) return false; + switch (a.type) { + case "text": { + const o = b as Extract<Chunk, { type: "text" }>; + return a.text === o.text; + } + case "thinking": { + const o = b as Extract<Chunk, { type: "thinking" }>; + return a.text === o.text; + } + case "image": { + const o = b as Extract<Chunk, { type: "image" }>; + return a.url === o.url; + } + case "error": { + const o = b as Extract<Chunk, { type: "error" }>; + return a.message === o.message && a.code === o.code; + } + case "system": { + const o = b as Extract<Chunk, { type: "system" }>; + return a.text === o.text; + } + case "tool-call": { + const o = b as Extract<Chunk, { type: "tool-call" }>; + return a.toolCallId === o.toolCallId && a.toolName === o.toolName; + } + case "tool-result": { + const o = b as Extract<Chunk, { type: "tool-result" }>; + return a.toolCallId === o.toolCallId && a.toolName === o.toolName && a.isError === o.isError; + } + default: + return assertChunkExhaustive(a) === assertChunkExhaustive(b); + } +} + +/** + * The trailing run of consecutive same-role provisional chunks at the end of + * `provisional` (the optimistic echo of one message). Returns the start/end + * indices `[start, end)` into `provisional` (empty if none). + */ +function trailingRun( + provisional: readonly ProvisionalChunk[], + role: Role, +): readonly ProvisionalChunk[] { + const end = provisional.length; + let start = end; + while (start > 0 && provisional[start - 1]?.role === role) start--; + return provisional.slice(start, end); +} + +/** * Merge authoritative seq-keyed chunks into the committed history. * Dedupes by seq (new wins), keeps seq-monotonic order, idempotent. * When sealedTurnId is set, drops all provisional chunks (now superseded) @@ -78,21 +136,30 @@ export function applyHistory( // During generation: if new committed chunks arrived, the provisional // array may contain duplicates — the optimistic echo from `appendUserMessage` - // is now backed by a committed chunk (CR-6: user message persisted at turn - // start). Remove provisional chunks that match the last committed chunk - // (role + chunk content), keeping only the accumulating (streaming) chunk. + // is now backed by committed chunks (CR-6: user message persisted at turn + // start). A user message may be multi-chunk (`[text, image, image, …]`), so + // match the trailing provisional user-run against the trailing committed + // user-run by content equality and drop the whole echo when it is fully + // backed. Leaves the accumulating (streaming) chunk untouched. if (addedNew && state.generating && state.provisional.length > 0) { - const lastCommitted = committed[committed.length - 1]; - if (lastCommitted !== undefined) { - const provisional = state.provisional.filter((p) => { - if (p.role !== lastCommitted.role) return true; - if (p.chunk.type !== lastCommitted.chunk.type) return true; - if (p.chunk.type === "text" && lastCommitted.chunk.type === "text") { - return p.chunk.text !== lastCommitted.chunk.text; - } - return true; - }); - return { ...state, committed, provisional, accumulating: state.accumulating }; + const provRun = trailingRun(state.provisional, "user"); + if (provRun.length > 0) { + const commRun = trailingRun(committed, "user"); + // Drop the provisional user echo iff every echoed chunk is content-equal + // to the corresponding committed chunk (the server persisted the same + // message). A partial match (echo longer than committed) keeps the echo + // — the not-yet-committed tail stays until the turn seals. + const fullyBacked = + commRun.length >= provRun.length && + provRun.every((p, i) => { + const c = commRun[i]; + return c !== undefined && chunkContentEquals(p.chunk, c.chunk); + }); + if (fullyBacked) { + const dropStart = state.provisional.length - provRun.length; + const provisional = state.provisional.slice(0, dropStart); + return { ...state, committed, provisional, accumulating: state.accumulating }; + } } } @@ -138,16 +205,18 @@ function reduceEvent(state: TranscriptState, event: AgentEvent): TranscriptState // The turn's USER prompt, surfaced on the event stream (backend CR-3) so a // WATCHER/late-joiner renders it mid-turn instead of waiting for seal. The // SENDER already echoed its own prompt optimistically (`appendUserMessage`), - // so DE-DUP: skip if the trailing provisional chunk is already an identical - // user text chunk. A pure watcher has no such echo → it appends and renders. + // so DE-DUP: skip if the trailing provisional USER run already contains an + // identical user text chunk. The echo may be multi-chunk (`[text, image, + // image, …]` — `appendUserMessage` appends the text first, then images), so + // we scan the whole trailing user run, not just the last chunk (which would + // be an image when images were pasted). A pure watcher has no such echo → + // it appends and renders. The `user-message` event carries ONLY text (never + // images — images arrive via history/loadSince); an images-only send (empty + // text) emits no `user-message` and is not de-duped here. if (event.text.length === 0) return state; - const last = state.provisional[state.provisional.length - 1]; - if ( - last !== undefined && - last.role === "user" && - last.chunk.type === "text" && - last.chunk.text === event.text - ) { + const run = trailingRun(state.provisional, "user"); + const alreadyEchoed = run.some((p) => p.chunk.type === "text" && p.chunk.text === event.text); + if (alreadyEchoed) { return { ...state, generating: true }; } const provisional = flushAccumulating(state.provisional, state.accumulating); @@ -335,15 +404,43 @@ export function foldEvent(state: TranscriptState, event: AgentEvent): Transcript /** * Optimistically append a user message to the provisional list. * Flushes any in-progress accumulating chunk first (defensively). - * The provisional user chunk is superseded when applyHistory receives + * The provisional user chunks are superseded when applyHistory receives * the authoritative committed chunks after a turn seals. + * + * When `images` are provided, they are appended AFTER the text chunk (in + * order) as `image` chunks — matching the server's persisted layout + * (`[text, image, image, …]` in one user message). A text chunk is only + * appended when `text` is non-empty (an images-only send echoes just the + * images). The `user-message` event carries only text (never images), so its + * de-dup scans the trailing user run for the echoed text rather than just the + * last chunk. */ -export function appendUserMessage(state: TranscriptState, text: string): TranscriptState { +export function appendUserMessage( + state: TranscriptState, + text: string, + images?: readonly ImageInput[], +): TranscriptState { const provisional = flushAccumulating(state.provisional, state.accumulating); - const userChunk: Chunk = { type: "text", text }; + const userChunks: Chunk[] = []; + if (text.length > 0) userChunks.push({ type: "text", text }); + if (images !== undefined) { + for (const img of images) { + if (img.url.length === 0) continue; + const chunk: Chunk = + img.mimeType !== undefined + ? { type: "image", url: img.url, mimeType: img.mimeType } + : { type: "image", url: img.url }; + userChunks.push(chunk); + } + } + if (userChunks.length === 0) { + // Nothing to echo (empty text + no images) — leave state unchanged but + // still flush any accumulating chunk defensively. + return { ...state, provisional, accumulating: null }; + } return { ...state, - provisional: [...provisional, { role: "user", chunk: userChunk }], + provisional: [...provisional, ...userChunks.map((chunk) => ({ role: "user", chunk }) as const)], accumulating: null, }; } diff --git a/src/core/wire/conformance.test.ts b/src/core/wire/conformance.test.ts index 880af07..0d955d5 100644 --- a/src/core/wire/conformance.test.ts +++ b/src/core/wire/conformance.test.ts @@ -128,9 +128,32 @@ describe("classifies every Chunk type", () => { }, { type: "error" as const, message: "e" }, { type: "system" as const, text: "s" }, + { type: "image" as const, url: "data:image/png;base64,AAAA", mimeType: "image/png" }, ]; const labels = chunks.map(assertChunkExhaustive); - expect(labels).toEqual(["text", "thinking", "tool-call", "tool-result", "error", "system"]); + expect(labels).toEqual([ + "text", + "thinking", + "tool-call", + "tool-result", + "error", + "system", + "image", + ]); + }); + + it("covers all 7 Chunk variants", () => { + // Keeps the exhaustive guard honest: a new Chunk.type variant must be added + // both here and to `assertChunkExhaustive` or the `satisfies never` errors. + expect([ + "text", + "thinking", + "tool-call", + "tool-result", + "error", + "system", + "image", + ] as const).toHaveLength(7); }); }); @@ -222,6 +245,21 @@ describe("ChatSendMessage shape is constructible", () => { expect(msg.model).toBe("default/gpt-4"); expect(msg.cwd).toBe("/tmp"); }); + + it("constructs a ChatSendMessage with pasted images", () => { + const msg: ChatSendMessage = { + type: "chat.send", + conversationId: "c1", + message: "what's in this image?", + images: [ + { url: "data:image/png;base64,AAAA", mimeType: "image/png" }, + { url: "https://example.com/cat.jpg" }, + ], + }; + expect(msg.images).toHaveLength(2); + expect(msg.images?.[0]?.mimeType).toBe("image/png"); + expect(msg.images?.[1]?.mimeType).toBeUndefined(); + }); }); describe("ConversationHistoryResponse shape is constructible", () => { diff --git a/src/core/wire/conformance.ts b/src/core/wire/conformance.ts index 412d80a..bfa67dc 100644 --- a/src/core/wire/conformance.ts +++ b/src/core/wire/conformance.ts @@ -60,6 +60,8 @@ export function assertChunkExhaustive(chunk: Chunk): string { return "error"; case "system": return "system"; + case "image": + return "image"; default: return chunk satisfies never; } diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts index 5419b89..cf57cea 100644 --- a/src/features/chat/index.ts +++ b/src/features/chat/index.ts @@ -4,8 +4,9 @@ export type { RenderGroup, ToolBatchEntry, } from "../../core/chunks"; -export { groupRenderedChunks, viewProviderRetry } from "../../core/chunks"; +export { groupRenderedChunks, resolveImageUrl, viewProviderRetry } from "../../core/chunks"; export type { TurnMetricsEntry } from "../../core/metrics"; +export { isVisionModel } from "./model-select"; export type { ChatTransport, HistorySync, HistoryWindow, MetricsSync } from "./ports"; export type { EffortOption, diff --git a/src/features/chat/model-select.test.ts b/src/features/chat/model-select.test.ts index deb673d..6d3081d 100644 --- a/src/features/chat/model-select.test.ts +++ b/src/features/chat/model-select.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; -import { joinModelName, modelKeys, modelsForKey, splitModelName } from "./model-select"; +import { + isVisionModel, + joinModelName, + modelKeys, + modelsForKey, + splitModelName, +} from "./model-select"; describe("splitModelName", () => { it("splits on the first slash", () => { @@ -56,3 +62,28 @@ describe("modelsForKey", () => { expect(modelsForKey(["openai/gpt-4"], "anthropic")).toEqual([]); }); }); + +describe("isVisionModel", () => { + it("returns true when modelInfo[name].vision is true", () => { + const info = { "kimi/k2": { vision: true } }; + expect(isVisionModel(info, "kimi/k2")).toBe(true); + }); + + it("returns false when vision is false", () => { + const info = { "umans/glm-5.2": { vision: false } }; + expect(isVisionModel(info, "umans/glm-5.2")).toBe(false); + }); + + it("returns false when vision is absent (unknown)", () => { + const info = { "umans/glm-5.2": { contextWindow: 128000 } }; + expect(isVisionModel(info, "umans/glm-5.2")).toBe(false); + }); + + it("returns false for a model with no metadata entry at all", () => { + expect(isVisionModel({}, "unknown/model")).toBe(false); + }); + + it("returns false for an empty modelInfo map", () => { + expect(isVisionModel({}, "kimi/k2")).toBe(false); + }); +}); diff --git a/src/features/chat/model-select.ts b/src/features/chat/model-select.ts index db41772..602e0ef 100644 --- a/src/features/chat/model-select.ts +++ b/src/features/chat/model-select.ts @@ -1,3 +1,5 @@ +import type { ModelMetadata } from "@dispatch/transport-contract"; + /** * Pure helpers for the two-step model picker. * @@ -47,3 +49,16 @@ export function modelsForKey(models: readonly string[], key: string): string[] { } return out; } + +/** + * Whether a given full model name (`<key>/<model>`) is vision-capable — i.e. + * `GET /models` `modelInfo[name].vision === true`. Absent/`false`/unknown → + * `false` (the server's vision handoff transcribes images to text for it). + * Pure lookup against the catalog metadata; zero DOM. + */ +export function isVisionModel( + modelInfo: Readonly<Record<string, ModelMetadata>>, + fullName: string, +): boolean { + return modelInfo[fullName]?.vision === true; +} diff --git a/src/features/chat/store.svelte.ts b/src/features/chat/store.svelte.ts index 3588da1..5278737 100644 --- a/src/features/chat/store.svelte.ts +++ b/src/features/chat/store.svelte.ts @@ -4,7 +4,7 @@ import type { ChatQueueMessage, ChatSendMessage, } from "@dispatch/transport-contract"; -import type { ChatMessage, StoredChunk, TurnProviderRetryEvent } from "@dispatch/wire"; +import type { ChatMessage, ImageInput, StoredChunk, TurnProviderRetryEvent } from "@dispatch/wire"; import type { RenderedChunk, TranscriptState } from "../../core/chunks"; import { appendUserMessage, @@ -109,7 +109,14 @@ export interface ChatStore { */ readonly thinkingKeyBase: number; handleDelta(msg: ChatDeltaMessage | ChatErrorMessage): void; - send(text: string): void; + /** + * Send a user message (start a turn via `chat.send`). Optimistically echoes + * the text + any `images` as provisional user chunks (`[text, image, …]` in + * order), then forwards them on the WS `chat.send` op. `images` is omitted on + * the wire when none are staged (text-only, backward compatible). An + * images-only send (empty text) is allowed — the message text is `""`. + */ + send(text: string, images?: readonly ImageInput[]): void; /** * Enqueue a steering message onto the conversation's queue (`chat.queue` * WS op). While a turn is generating, the message is delivered mid-turn at @@ -312,8 +319,8 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { } }, - send(text: string): void { - transcript = appendUserMessage(transcript, text); + send(text: string, images?: readonly ImageInput[]): void { + transcript = appendUserMessage(transcript, text, images); maybeTrim(); const msg: ChatSendMessage = { type: "chat.send", @@ -321,6 +328,7 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { message: text, ...(_model !== undefined ? { model: _model } : {}), ...(deps.workspaceId !== undefined ? { workspaceId: deps.workspaceId } : {}), + ...(images !== undefined && images.length > 0 ? { images: [...images] } : {}), }; deps.transport.send(msg); }, diff --git a/src/features/chat/store.test.ts b/src/features/chat/store.test.ts index c052d1b..8f36994 100644 --- a/src/features/chat/store.test.ts +++ b/src/features/chat/store.test.ts @@ -1,4 +1,4 @@ -import type { AgentEvent, StepId, StoredChunk } from "@dispatch/wire"; +import type { AgentEvent, ImageInput, StepId, StoredChunk } from "@dispatch/wire"; import { describe, expect, it, vi } from "vitest"; import { createChatStore } from "./store.svelte"; import { @@ -144,6 +144,93 @@ describe("createChatStore", () => { store.dispose(); }); + it("send forwards staged images on chat.send and echoes them provisionally", () => { + const transport = createFakeTransport(); + const historySync = createFakeHistorySync(); + const metricsSync = createFakeMetricsSync(); + const cache = createFakeCache(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: historySync.impl, + metricsSync: metricsSync.impl, + cache: cache.impl, + }); + + const images: ImageInput[] = [ + { url: "data:image/png;base64,AAAA", mimeType: "image/png" }, + { url: "https://example.com/cat.jpg" }, + ]; + store.send("look at this", images); + + expect(transport.sent).toHaveLength(1); + const msg = transport.sent[0]; + expect(msg?.type).toBe("chat.send"); + expect(msg?.message).toBe("look at this"); + expect(msg?.images).toEqual(images); + + // Optimistic echo: a text chunk + two image chunks, provisional. + const chunks = store.chunks; + expect(chunks).toHaveLength(3); + expect(chunks[0]?.chunk).toEqual({ type: "text", text: "look at this" }); + expect(chunks[1]?.chunk).toEqual({ type: "image", url: images[0]?.url, mimeType: "image/png" }); + expect(chunks[2]?.chunk).toEqual({ type: "image", url: images[1]?.url }); + + store.dispose(); + }); + + it("send omits images on the wire when none are staged (backward compatible)", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("just text"); + + expect(transport.sent[0]).not.toHaveProperty("images"); + store.dispose(); + }); + + it("send omits images on the wire for an empty array", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("just text", []); + + expect(transport.sent[0]).not.toHaveProperty("images"); + store.dispose(); + }); + + it("send allows an images-only message (empty text)", () => { + const transport = createFakeTransport(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: createFakeHistorySync().impl, + metricsSync: createFakeMetricsSync().impl, + cache: createFakeCache().impl, + }); + + store.send("", [{ url: "data:image/png;base64,AAAA", mimeType: "image/png" }]); + + expect(transport.sent[0]?.message).toBe(""); + expect(transport.sent[0]?.images).toHaveLength(1); + // The echo is image-only (no text chunk). + expect(store.chunks).toHaveLength(1); + expect(store.chunks[0]?.chunk.type).toBe("image"); + store.dispose(); + }); + describe("queueMessage (chat.queue — steering)", () => { it("posts a chat.queue with conversationId + text", () => { const transport = createFakeTransport(); diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index a2fd944..5f8067d 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -607,6 +607,180 @@ describe("ChatView", () => { // Turn total should NOT render (total is null — turn still in progress) expect(screen.queryByText(/^turn/)).toBeNull(); }); + + it("renders a user image chunk as an <img> with the chunk's url", () => { + const url = "data:image/png;base64,AAAA"; + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url, mimeType: "image/png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { props: { chunks } }); + + const img = container.querySelector("img"); + expect(img).not.toBeNull(); + expect(img?.getAttribute("src")).toBe(url); + expect(img?.getAttribute("alt")).toBe("image/png"); + expect(img?.getAttribute("loading")).toBe("lazy"); + }); + + it("resolves a persisted image chunk's relative url against apiBaseUrl", () => { + // Persisted image chunks now carry a compact relative path (`/images/…`) + // served by the backend — prepend the API base to render them. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url: "/images/conv-123/abc-456.png", mimeType: "image/png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe( + "http://localhost:24203/images/conv-123/abc-456.png", + ); + }); + + it("passes a data URL through unchanged even with apiBaseUrl set (optimistic echo)", () => { + // The optimistic echo (what the FE just sent) is still a data URL; it must + // NOT be mangled by the base-URL prepend. + const dataUrl = "data:image/png;base64,iVBOR="; + const chunks: RenderedChunk[] = [ + { seq: null, role: "user", chunk: { type: "image", url: dataUrl }, provisional: true }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe(dataUrl); + }); + + it("leaves a relative image url root-relative when apiBaseUrl is absent", () => { + // No apiBaseUrl → a browser resolves `/images/…` against the document origin. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url: "/images/conv-1/x.png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { props: { chunks } }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe("/images/conv-1/x.png"); + }); + + it("renders a multi-chunk user message [text, image] and a transcription text", () => { + // A non-vision model: the server persists the original image chunk AND a + // transcription text chunk in the SAME user message — render both. + const url = "data:image/png;base64,BBQ="; + const chunks: RenderedChunk[] = [ + { seq: 1, role: "user", chunk: { type: "text", text: "describe this" }, provisional: false }, + { + seq: 2, + role: "user", + chunk: { type: "image", url, mimeType: "image/png" }, + provisional: false, + }, + { + seq: 3, + role: "user", + chunk: { type: "text", text: "[Image analysis (via kimi/k2)]: a red square" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { props: { chunks } }); + + expect(screen.getByText("describe this")).toBeInTheDocument(); + expect(screen.getByText(/\[Image analysis/)).toBeInTheDocument(); + expect(container.querySelector("img")?.getAttribute("src")).toBe(url); + }); + + it("renders a consult_vision tool call/result like any other tool", () => { + // read_image is GONE — replaced by consult_vision (opens a vision-model + // conversation, attaches the image + question, returns the answer). It is + // a normal tool call: rendered generically by toolName. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "assistant", + chunk: { + type: "tool-call", + toolCallId: "tc1", + toolName: "consult_vision", + input: { question: "what is in this image?", imageIds: [1] }, + }, + provisional: false, + }, + { + seq: 2, + role: "tool", + chunk: { + type: "tool-result", + toolCallId: "tc1", + toolName: "consult_vision", + content: "a red square on a white background", + isError: false, + }, + provisional: false, + }, + ]; + + render(ChatView, { props: { chunks } }); + + expect(screen.getAllByText("consult_vision").length).toBeGreaterThan(0); + expect(screen.getByText("a red square on a white background")).toBeInTheDocument(); + }); + + it("renders a non-vision placeholder text chunk as-is", () => { + // A non-vision model gets a numbered placeholder (a regular text chunk) + // instead of an auto-transcription. Renders like any text chunk. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { + type: "text", + text: "[Image 1 attached — call consult_vision with imageIds=[1] and a specific question to analyze it]", + }, + provisional: false, + }, + ]; + + render(ChatView, { props: { chunks } }); + + expect(screen.getByText(/\[Image 1 attached/)).toBeInTheDocument(); + expect(screen.getByText(/consult_vision with imageIds/)).toBeInTheDocument(); + }); + + it("renders a compacted-image text chunk as-is", () => { + // Image compaction transcribes old images to [Compacted image]: <desc>. + // Regular text chunk — render as-is. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "text", text: "[Compacted image]: a chart showing rising sales" }, + provisional: false, + }, + ]; + + render(ChatView, { props: { chunks } }); + + expect(screen.getByText(/\[Compacted image\]/)).toBeInTheDocument(); + expect(screen.getByText(/rising sales/)).toBeInTheDocument(); + }); }); describe("Composer", () => { @@ -623,7 +797,7 @@ describe("Composer", () => { await user.click(sendButton); expect(onSend).toHaveBeenCalledTimes(1); - expect(onSend).toHaveBeenCalledWith("Hello world"); + expect(onSend).toHaveBeenCalledWith("Hello world", undefined); expect(textarea).toHaveValue(""); }); @@ -651,7 +825,7 @@ describe("Composer", () => { const sendButton = screen.getByRole("button", { name: "Send" }); await user.click(sendButton); - expect(onSend).toHaveBeenCalledWith("hello"); + expect(onSend).toHaveBeenCalledWith("hello", undefined); }); it("sends on Enter key (without Shift)", async () => { @@ -663,7 +837,7 @@ describe("Composer", () => { const textarea = screen.getByRole("textbox", { name: "Message input" }); await user.type(textarea, "Test message{Enter}"); - expect(onSend).toHaveBeenCalledWith("Test message"); + expect(onSend).toHaveBeenCalledWith("Test message", undefined); }); it("does not send on Shift+Enter", async () => { @@ -677,6 +851,142 @@ describe("Composer", () => { expect(onSend).not.toHaveBeenCalled(); }); + + it("stages a pasted image and forwards it on send", async () => { + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend } }); + + const textarea = screen.getByRole("textbox", { name: "Message input" }); + await user.type(textarea, "look at this"); + + // jsdom has no ClipboardEvent/DataTransfer: dispatch a plain paste event + // carrying a mock clipboardData whose only item is an image File. + const file = new File(["PNG"], "shot.png", { type: "image/png" }); + const paste = new Event("paste", { bubbles: true }); + Object.defineProperty(paste, "clipboardData", { + value: { + items: [{ kind: "file", type: "image/png", getAsFile: () => file }], + }, + }); + container.querySelector("textarea")?.dispatchEvent(paste); + + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: "Send" })); + + expect(onSend).toHaveBeenCalledTimes(1); + const [, images] = onSend.mock.calls[0] ?? []; + expect(images).toHaveLength(1); + expect(images[0]?.url).toMatch(/^data:image\/png;base64,/); + expect(images[0]?.mimeType).toBe("image/png"); + }); + + it("lets a text paste proceed when no image is on the clipboard", async () => { + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend } }); + + const textarea = screen.getByRole("textbox", { name: "Message input" }); + await user.type(textarea, "hello"); + + // A text-only paste: no file items → the component must NOT preventDefault, + // so the default text paste path is unaffected (no image staged). + const paste = new Event("paste", { bubbles: true }); + Object.defineProperty(paste, "clipboardData", { + value: { items: [{ kind: "string", type: "text/plain" }] }, + }); + container.querySelector("textarea")?.dispatchEvent(paste); + + await new Promise((r) => setTimeout(r, 0)); + expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument(); + }); + + it("stages an image via the attach button's file picker", async () => { + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend } }); + + const file = new File(["JPG"], "photo.jpg", { type: "image/jpeg" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + + // Image-only send (no text): the Send button is enabled. + const send = screen.getByRole("button", { name: "Send" }); + expect(send).not.toBeDisabled(); + await user.click(send); + + expect(onSend).toHaveBeenCalledTimes(1); + const [text, images] = onSend.mock.calls[0] ?? []; + expect(text).toBe(""); + expect(images).toHaveLength(1); + expect(images[0]?.mimeType).toBe("image/jpeg"); + }); + + it("removes a staged image via the remove button", async () => { + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend } }); + + const file = new File(["PNG"], "shot.png", { type: "image/png" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: "Remove image" })); + + expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument(); + // With no text and no images, Send is disabled again. + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); + }); + + it("ignores a non-image file chosen via the picker", async () => { + const onSend = vi.fn(); + const { container } = render(Composer, { props: { onSend } }); + + const file = new File(["TXT"], "notes.txt", { type: "text/plain" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + + // Give the async staging a chance; a non-image is skipped. + await new Promise((r) => setTimeout(r, 0)); + expect(screen.queryByRole("button", { name: "Remove image" })).not.toBeInTheDocument(); + }); + + it("queues (steers) text-only and never forwards images", async () => { + // While running, the Send button becomes "Queue"; steering is text-only. + const onQueue = vi.fn(); + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend, onQueue, status: "running" } }); + + const textarea = screen.getByRole("textbox", { name: "Message input" }); + await user.type(textarea, "steer here"); + + // Also stage an image — it must NOT be forwarded on a queue. + const file = new File(["PNG"], "shot.png", { type: "image/png" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: "Queue" })); + expect(onQueue).toHaveBeenCalledWith("steer here"); + expect(onSend).not.toHaveBeenCalled(); + }); }); describe("ModelSelector", () => { @@ -730,6 +1040,55 @@ describe("ModelSelector", () => { expect(onSelect).toHaveBeenCalledTimes(1); expect(onSelect).toHaveBeenCalledWith("openai/gpt-4o"); }); + + it("marks vision-capable models in the model dropdown", () => { + const models = ["kimi/k2", "kimi/k1.5"]; + const modelInfo = { + "kimi/k2": { vision: true }, + "kimi/k1.5": { vision: false }, + }; + render(ModelSelector, { + props: { models, selected: "kimi/k2", onSelect: vi.fn(), modelInfo }, + }); + + const modelSelect = screen.getByRole("combobox", { name: "Model selector" }); + const options = within(modelSelect).getAllByRole("option"); + expect(options).toHaveLength(2); + expect(options[0]?.textContent).toContain("vision"); + expect(options[1]?.textContent).not.toContain("vision"); + }); + + it("shows the vision indicator when the selected model is vision-capable", () => { + render(ModelSelector, { + props: { + models: ["kimi/k2"], + selected: "kimi/k2", + onSelect: vi.fn(), + modelInfo: { "kimi/k2": { vision: true } }, + }, + }); + expect(screen.getByText(/sees images natively/)).toBeInTheDocument(); + }); + + it("shows the vision-handoff hint when the selected model is non-vision", () => { + render(ModelSelector, { + props: { + models: ["umans/glm-5.2"], + selected: "umans/glm-5.2", + onSelect: vi.fn(), + modelInfo: { "umans/glm-5.2": { vision: false } }, + }, + }); + expect(screen.getByText(/auto-described/i)).toBeInTheDocument(); + expect(screen.queryByText(/sees images natively/)).not.toBeInTheDocument(); + }); + + it("shows the handoff hint when modelInfo is absent", () => { + render(ModelSelector, { + props: { models: ["openai/gpt-4"], selected: "openai/gpt-4", onSelect: vi.fn() }, + }); + expect(screen.getByText(/auto-described/i)).toBeInTheDocument(); + }); }); describe("ReasoningEffortSelector", () => { diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte index b49f7b4..e67ca5b 100644 --- a/src/features/chat/ui/ChatView.svelte +++ b/src/features/chat/ui/ChatView.svelte @@ -1,6 +1,6 @@ <script lang="ts"> import type { TurnProviderRetryEvent } from "@dispatch/wire"; - import { groupRenderedChunks, type RenderedChunk, viewProviderRetry } from "../index"; + import { groupRenderedChunks, resolveImageUrl, type RenderedChunk, viewProviderRetry } from "../index"; import { interleaveTurnMetrics, viewCacheRate, @@ -24,6 +24,7 @@ onShowEarlier, thinkingKeyBase = 0, providerRetry = null, + apiBaseUrl = "", }: { chunks: readonly RenderedChunk[]; turnMetrics?: readonly TurnMetricsEntry[]; @@ -44,6 +45,14 @@ * newest attempt + delay, and is cleared when content resumes / turn ends. */ providerRetry?: TurnProviderRetryEvent | null; + /** + * The HTTP API base URL (e.g. `http://localhost:24203`). Persisted image + * chunks carry a compact relative path (`/images/<conv>/<uuid>.png`); this + * base is prepended to render them. The optimistic echo's data URL and any + * absolute URL pass through unchanged (see `resolveImageUrl`). Defaults to + * "" (root-relative — a browser resolves `/images/…` against its origin). + */ + apiBaseUrl?: string; } = $props(); // True while a show-earlier page-in is awaited (disables the button). @@ -95,11 +104,23 @@ {#snippet chunkRow(rendered: RenderedChunk)} {#if rendered.role === "user"} - <!-- User: a speech bubble, left-aligned --> + <!-- User: a speech bubble, left-aligned. A user message may be multi-chunk + ([text, image, image, …]); each chunk renders in its own bubble. A + persisted image chunk's url is a compact relative path (`/images/…`) + served by the backend — resolve it against the API base. The + optimistic echo's data URL (and any absolute URL) passes through. --> <div class="chat chat-start"> <div class="chat-bubble chat-bubble-primary"> {#if rendered.chunk.type === "text"} <p>{rendered.chunk.text}</p> + {:else if rendered.chunk.type === "image"} + <img + src={resolveImageUrl(rendered.chunk.url, apiBaseUrl)} + alt={rendered.chunk.mimeType ?? "pasted image"} + loading="lazy" + decoding="async" + class="max-h-80 max-w-full rounded" + /> {/if} </div> </div> diff --git a/src/features/chat/ui/Composer.svelte b/src/features/chat/ui/Composer.svelte index 0327f93..afe1e3c 100644 --- a/src/features/chat/ui/Composer.svelte +++ b/src/features/chat/ui/Composer.svelte @@ -1,8 +1,19 @@ <script lang="ts"> + import type { ImageInput } from "@dispatch/wire"; import { computeContextUsage, formatCompactTokens } from "../../../core/metrics"; const FALLBACK_CONTEXT_WINDOW = 1_000_000; const MAX_LINES = 7; + /** Accept only raster images (the provider image-content formats). */ + const IMAGE_ACCEPT = "image/png,image/jpeg,image/gif,image/webp"; + /** Reject images larger than this before base64-encoding (keeps payloads sane). */ + const MAX_IMAGE_BYTES = 8 * 1024 * 1024; + + /** A staged image awaiting send: a stable id + the `ImageInput` to forward. */ + interface StagedImage { + readonly id: string; + readonly input: ImageInput; + } let { onSend, @@ -12,12 +23,18 @@ contextWindow = undefined, status = "idle", }: { - onSend: (text: string) => void; + /** + * Send a message (start a turn via `chat.send`). Carries any staged images + * as `ImageInput[]` (base64 data URLs or https URLs); the store forwards + * them on the WS `chat.send` op / `POST /chat` body. `images` is omitted + * (not an empty array) when none are staged, so the wire stays text-only. + */ + onSend: (text: string, images?: ImageInput[]) => void; /** * Enqueue a steering message (`chat.queue`). When provided AND the status * is `running`, the send button becomes a "Queue" button that steers the - * in-flight turn instead of starting a new one. When absent, `onSend` is - * used regardless (tests / non-steering contexts). + * in-flight turn instead of starting a new one. Steering is text-only — + * it never carries images (a mid-turn injection has no image surface). */ onQueue?: (text: string) => void; /** Stop the in-flight generation (`POST /conversations/:id/stop`). */ @@ -39,22 +56,30 @@ export type ComposerStatus = "idle" | "running" | "queued" | "error"; let text = $state(""); + let images = $state<StagedImage[]>([]); let inputEl: HTMLTextAreaElement | undefined; + let fileInputEl: HTMLInputElement | undefined; + let dragOver = $state(false); const hasText = $derived(text.trim().length > 0); + const hasImages = $derived(images.length > 0); + const canSend = $derived(hasText || hasImages); const effectiveMax = $derived(contextWindow ?? FALLBACK_CONTEXT_WINDOW); const usage = $derived(computeContextUsage(contextSize, effectiveMax)); const hasUsage = $derived(contextSize !== undefined); // One button, three modes: // - idle → "Send" (starts a turn via chat.send) - // - running/queued + text → "Queue" (steers via chat.queue) + // - running/queued + text → "Queue" (steers via chat.queue — text only) // - running/queued + empty → "Stop" (aborts via POST /stop) // (`queued` behaves like `running` — the turn is in flight, just waiting for a // concurrency slot; the user can still steer or stop it.) + // Steering never carries images: when running with images staged but no text, + // the images stay staged (queue is text-only). Images-without-text while running + // is an unusual case that still sends (the server auto-starts/resolves). const inFlight = $derived(status === "running" || status === "queued"); const buttonMode = $derived.by<"send" | "queue" | "stop">(() => { - if (inFlight && !hasText && onStop !== undefined) return "stop"; + if (inFlight && !hasText && !hasImages && onStop !== undefined) return "stop"; if (inFlight && hasText && onQueue !== undefined) return "queue"; return "send"; }); @@ -63,7 +88,7 @@ ? "Queued for a slot…" : status === "running" ? "Steer the conversation..." - : "Type a message...", + : "Type a message, paste or drop an image…", ); // As the window fills, escalate color: calm → warning → danger. @@ -94,15 +119,97 @@ resize(); }); + /** Read a File into a base64 data URL (`data:image/…;base64,…`). */ + function fileToDataUrl(file: File): Promise<string> { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + if (typeof reader.result === "string") resolve(reader.result); + else reject(new Error("unreadable image")); + }; + reader.onerror = () => reject(reader.error ?? new Error("read failed")); + reader.readAsDataURL(file); + }); + } + + let imgSeq = 0; + /** Stage a File as an image (skip non-images / oversized). Returns whether staged. */ + async function stageFile(file: File): Promise<boolean> { + if (!file.type.startsWith("image/")) return false; + if (file.size > MAX_IMAGE_BYTES) return false; + const url = await fileToDataUrl(file); + // Prefer the file's declared MIME; fall back to the data URL's prefix. + const mimeType = file.type || undefined; + const id = `img-${Date.now()}-${imgSeq++}`; + images = [...images, { id, input: { url, ...(mimeType ? { mimeType } : {}) } }]; + return true; + } + + function removeImage(id: string): void { + images = images.filter((img) => img.id !== id); + } + + /** Handle a paste anywhere in the form: extract image items from the clipboard. */ + async function handlePaste(e: ClipboardEvent): Promise<void> { + const items = e.clipboardData?.items; + if (items === undefined) return; + let hadImage = false; + const staged: File[] = []; + for (const item of items) { + if (item.kind === "file" && item.type.startsWith("image/")) { + const file = item.getAsFile(); + if (file !== null) { + staged.push(file); + hadImage = true; + } + } + } + if (!hadImage) return; // let the default text paste proceed + e.preventDefault(); // suppress pasting the image as a filename string + for (const file of staged) { + await stageFile(file); + } + } + + /** File-picker <input type="file"> change. */ + async function handleFilePick(e: Event): Promise<void> { + const target = e.currentTarget as HTMLInputElement; + const files = target.files; + if (files === null) return; + for (const file of files) { + await stageFile(file); + } + target.value = ""; // reset so picking the same file again re-fires change + } + + /** Drop images onto the composer. */ + async function handleDrop(e: DragEvent): Promise<void> { + dragOver = false; + const files = e.dataTransfer?.files; + if (files === undefined || files.length === 0) return; + const hadImage = Array.from(files).some((f) => f.type.startsWith("image/")); + if (!hadImage) return; + e.preventDefault(); + for (const file of files) { + await stageFile(file); + } + } + function handleSubmit(): void { const trimmed = text.trim(); - if (trimmed.length === 0) return; + // Allow a send with images even when text is empty (an image-only turn). + if (trimmed.length === 0 && !hasImages) return; if (buttonMode === "queue") { + // Steering is text-only — never forward images. onQueue?.(trimmed); } else { - onSend(trimmed); + const toSend: ImageInput[] | undefined = hasImages + ? images.map((img) => img.input) + : undefined; + onSend(trimmed, toSend); } text = ""; + images = []; } function handleKeydown(e: KeyboardEvent): void { @@ -119,17 +226,68 @@ e.preventDefault(); handleSubmit(); }} + ondrop={handleDrop} + ondragover={(e) => { + if (e.dataTransfer?.types?.includes("Files")) { + e.preventDefault(); + dragOver = true; + } + }} + ondragleave={() => (dragOver = false)} > - <!-- Top bar: expanding textarea + single context-aware button --> + <!-- Top bar: expanding textarea + image-attach button + single context-aware button --> <div class="flex items-end gap-2 px-4 pt-3 pb-2"> - <textarea - bind:this={inputEl} - class="textarea textarea-bordered flex-1 resize-none leading-normal !min-h-0 h-auto" - bind:value={text} - onkeydown={handleKeydown} - {placeholder} - rows="1" - aria-label="Message input"></textarea> + <div + class="flex-1" + onpaste={handlePaste} + class:border-2={dragOver} + class:border-primary={dragOver} + class:border-dashed={dragOver} + class:rounded={dragOver} + > + <textarea + bind:this={inputEl} + class="textarea textarea-bordered w-full resize-none leading-normal !min-h-0 h-auto" + bind:value={text} + onkeydown={handleKeydown} + {placeholder} + rows="1" + aria-label="Message input"></textarea> + </div> + + <!-- Hidden file picker (images only; multiple). --> + <input + bind:this={fileInputEl} + type="file" + accept={IMAGE_ACCEPT} + multiple + class="hidden" + onchange={handleFilePick} + /> + <!-- Attach image button (opens the file picker). --> + <button + class="btn btn-ghost btn-square shrink-0" + type="button" + aria-label="Attach image" + title="Attach image" + onclick={() => fileInputEl?.click()} + > + <svg + xmlns="http://www.w3.org/2000/svg" + viewBox="0 0 24 24" + fill="none" + stroke="currentColor" + stroke-width="2" + stroke-linecap="round" + stroke-linejoin="round" + class="h-5 w-5" + > + <rect x="3" y="3" width="18" height="18" rx="2" ry="2"></rect> + <circle cx="8.5" cy="8.5" r="1.5"></circle> + <polyline points="21 15 16 10 5 21"></polyline> + </svg> + </button> + {#if buttonMode === "stop"} <button class="btn btn-error w-20 shrink-0" @@ -140,12 +298,47 @@ Stop </button> {:else} - <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!hasText}> + <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!canSend}> {buttonMode === "queue" ? "Queue" : "Send"} </button> {/if} </div> + <!-- Staged image thumbnails (previews) with remove buttons. --> + {#if hasImages} + <div class="flex flex-wrap gap-2 px-4 pb-1"> + {#each images as img (img.id)} + <div class="group relative h-20 w-20 shrink-0 overflow-hidden rounded border border-base-300"> + <img + src={img.input.url} + alt={img.input.mimeType ?? "staged image"} + class="h-full w-full object-cover" + /> + <button + class="btn btn-circle btn-xs absolute right-0 top-0 bg-base-100/80 hover:bg-error hover:text-error-content" + type="button" + aria-label="Remove image" + onclick={() => removeImage(img.id)} + > + <svg + xmlns="http://www.w3.org/2000/svg" + viewBox="0 0 24 24" + fill="none" + stroke="currentColor" + stroke-width="3" + stroke-linecap="round" + stroke-linejoin="round" + class="h-3 w-3" + > + <line x1="18" y1="6" x2="6" y2="18"></line> + <line x1="6" y1="6" x2="18" y2="18"></line> + </svg> + </button> + </div> + {/each} + </div> + {/if} + <!-- Bottom status bar: status icon · context-window fill · token count --> <div class="flex items-center gap-2 px-4 pb-2 text-xs text-base-content/50"> <span class="shrink-0"> diff --git a/src/features/chat/ui/ModelSelector.svelte b/src/features/chat/ui/ModelSelector.svelte index 03acb79..11b9feb 100644 --- a/src/features/chat/ui/ModelSelector.svelte +++ b/src/features/chat/ui/ModelSelector.svelte @@ -1,20 +1,32 @@ <script lang="ts"> - import { joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select"; + import type { ModelMetadata } from "@dispatch/transport-contract"; + import { isVisionModel, joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select"; let { models, selected, onSelect, + modelInfo = {}, }: { models: readonly string[]; selected: string; onSelect: (model: string) => void; + /** + * Per-model metadata from `GET /models` (`{ [name]: ModelMetadata }`). + * Used to show a "vision" badge next to models with `vision: true` (they + * natively accept images; others rely on the server's vision handoff). + * Optional — absent metadata → no badge (treated as non-vision). + */ + modelInfo?: Readonly<Record<string, ModelMetadata>>; } = $props(); const keys = $derived(modelKeys(models)); const current = $derived(splitModelName(selected)); const keyModels = $derived(modelsForKey(models, current.key)); + // Whether the currently-selected full model name is vision-capable. + const selectedVision = $derived(isVisionModel(modelInfo, selected)); + // Switching key jumps to the first model available under it. function selectKey(key: string): void { const first = modelsForKey(models, key)[0] ?? ""; @@ -24,6 +36,11 @@ function selectModel(model: string): void { onSelect(joinModelName(current.key, model)); } + + // The full `<key>/<model>` name for a model suffix under the current key. + function fullNameFor(modelSuffix: string): string { + return joinModelName(current.key, modelSuffix); + } </script> <div class="flex flex-col gap-2"> @@ -44,7 +61,32 @@ aria-label="Model selector" > {#each keyModels as model (model)} - <option value={model}>{model}</option> + <option value={model}> + {model}{#if isVisionModel(modelInfo, fullNameFor(model))} · vision{/if} + </option> {/each} </select> + {#if selectedVision} + <div class="flex items-center gap-1 text-xs text-base-content/60"> + <svg + xmlns="http://www.w3.org/2000/svg" + viewBox="0 0 24 24" + fill="none" + stroke="currentColor" + stroke-width="2" + stroke-linecap="round" + stroke-linejoin="round" + class="h-3.5 w-3.5" + aria-hidden="true" + > + <path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"></path> + <circle cx="12" cy="12" r="3"></circle> + </svg> + <span>Vision — this model sees images natively</span> + </div> + {:else} + <div class="text-xs text-base-content/40"> + Pasted images are auto-described (vision handoff) + </div> + {/if} </div> diff --git a/src/features/heartbeat/ui/RunModal.svelte b/src/features/heartbeat/ui/RunModal.svelte index a4ed356..92068ae 100644 --- a/src/features/heartbeat/ui/RunModal.svelte +++ b/src/features/heartbeat/ui/RunModal.svelte @@ -11,6 +11,7 @@ closeChat, stopRun, onClose, + apiBaseUrl = "", }: { /** The run to display (its conversation's chat is shown live). */ run: HeartbeatRunView; @@ -26,6 +27,11 @@ /** Stop the heartbeat run (`POST .../runs/:runId/stop`). */ stopRun: StopHeartbeatRun; onClose: () => void; + /** + * The HTTP API base URL, to resolve persisted image chunk URLs + * (`/images/…`) in the run's transcript. Defaults to "" (root-relative). + */ + apiBaseUrl?: string; } = $props(); // Open the live watch ONCE on mount (the modal is keyed per run.id, so a run @@ -153,6 +159,7 @@ onShowEarlier={chat.showEarlier} thinkingKeyBase={chat.thinkingKeyBase} providerRetry={chat.providerRetry} + apiBaseUrl={apiBaseUrl} /> {/if} </div> diff --git a/src/features/vision/index.ts b/src/features/vision/index.ts new file mode 100644 index 0000000..6956d75 --- /dev/null +++ b/src/features/vision/index.ts @@ -0,0 +1,30 @@ +export type { + CompactionModelOption, + ImageLimitParse, + LoadVisionSettings, + LoadVisionSettingsResult, + SaveVisionSettings, + SaveVisionSettingsResult, + VisionSettings, + VisionSettingsPatch, +} from "./logic/view-model"; +export { + AUTO_COMPACTION_MODEL, + compactionModelChanged, + compactionModelFromValue, + compactionModelOptions, + DEFAULT_IMAGE_LIMIT, + imageLimitChanged, + imageLimitLabel, + MAX_IMAGE_LIMIT, + normalizeVisionSettings, + parseImageLimit, + selectedCompactionValue, +} from "./logic/view-model"; +export { default as VisionSettingsView } from "./ui/VisionSettingsView.svelte"; + +/** Public module manifest — aggregated by the shell's "Loaded Modules" view. */ +export const manifest = { + name: "vision", + description: "Global vision settings (image compaction limit + model)", +} as const; diff --git a/src/features/vision/logic/view-model.test.ts b/src/features/vision/logic/view-model.test.ts new file mode 100644 index 0000000..b1db822 --- /dev/null +++ b/src/features/vision/logic/view-model.test.ts @@ -0,0 +1,198 @@ +import type { ModelMetadata } from "@dispatch/transport-contract"; +import { describe, expect, it } from "vitest"; +import { + AUTO_COMPACTION_MODEL, + compactionModelChanged, + compactionModelFromValue, + compactionModelOptions, + DEFAULT_IMAGE_LIMIT, + imageLimitChanged, + imageLimitLabel, + MAX_IMAGE_LIMIT, + normalizeVisionSettings, + parseImageLimit, + selectedCompactionValue, +} from "./view-model"; + +describe("normalizeVisionSettings", () => { + it("returns the value as-is for a well-formed body", () => { + expect(normalizeVisionSettings({ imageLimit: 5, compactionModel: "kimi/k2" })).toEqual({ + imageLimit: 5, + compactionModel: "kimi/k2", + }); + }); + + it("coerces a null compactionModel to null", () => { + expect(normalizeVisionSettings({ imageLimit: 10, compactionModel: null })).toEqual({ + imageLimit: 10, + compactionModel: null, + }); + }); + + it("defaults imageLimit when absent or non-numeric", () => { + expect(normalizeVisionSettings({ compactionModel: "kimi/k2" })).toEqual({ + imageLimit: DEFAULT_IMAGE_LIMIT, + compactionModel: "kimi/k2", + }); + expect(normalizeVisionSettings({ imageLimit: "oops", compactionModel: null })).toEqual({ + imageLimit: DEFAULT_IMAGE_LIMIT, + compactionModel: null, + }); + }); + + it("floors a fractional imageLimit", () => { + expect(normalizeVisionSettings({ imageLimit: 7.9, compactionModel: null }).imageLimit).toBe(7); + }); + + it("defaults for a null/non-object body (never crashes)", () => { + expect(normalizeVisionSettings(null)).toEqual({ + imageLimit: DEFAULT_IMAGE_LIMIT, + compactionModel: null, + }); + expect(normalizeVisionSettings("nope")).toEqual({ + imageLimit: DEFAULT_IMAGE_LIMIT, + compactionModel: null, + }); + }); + + it("treats a negative imageLimit as the default", () => { + expect(normalizeVisionSettings({ imageLimit: -3, compactionModel: null }).imageLimit).toBe( + DEFAULT_IMAGE_LIMIT, + ); + }); + + it("treats an empty compactionModel string as null", () => { + expect( + normalizeVisionSettings({ imageLimit: 10, compactionModel: "" }).compactionModel, + ).toBeNull(); + }); +}); + +describe("parseImageLimit", () => { + it("parses a non-negative integer", () => { + expect(parseImageLimit("5")).toEqual({ ok: true, value: 5 }); + expect(parseImageLimit("0")).toEqual({ ok: true, value: 0 }); + }); + + it("floors a fractional value", () => { + expect(parseImageLimit("7.9")).toEqual({ ok: true, value: 7 }); + }); + + it("trims whitespace", () => { + expect(parseImageLimit(" 12 ")).toEqual({ ok: true, value: 12 }); + }); + + it("errors on empty input", () => { + expect(parseImageLimit("")).toEqual({ ok: false, error: "Enter a number." }); + }); + + it("errors on non-numeric input", () => { + expect(parseImageLimit("lots").ok).toBe(false); + }); + + it("errors on a negative value", () => { + expect(parseImageLimit("-1").ok).toBe(false); + }); + + it("errors when above the max", () => { + expect(parseImageLimit(String(MAX_IMAGE_LIMIT + 1)).ok).toBe(false); + }); + + it("accepts the max", () => { + expect(parseImageLimit(String(MAX_IMAGE_LIMIT))).toEqual({ ok: true, value: MAX_IMAGE_LIMIT }); + }); +}); + +describe("imageLimitChanged", () => { + it("is true when the typed value differs after normalization", () => { + expect(imageLimitChanged("5", 10)).toBe(true); + }); + + it("is false when equal", () => { + expect(imageLimitChanged("10", 10)).toBe(false); + }); + + it("is false for invalid input", () => { + expect(imageLimitChanged("abc", 10)).toBe(false); + expect(imageLimitChanged("", 10)).toBe(false); + }); + + it("compares after flooring", () => { + expect(imageLimitChanged("7.9", 7)).toBe(false); + }); +}); + +describe("compactionModelOptions", () => { + const modelInfo: Record<string, ModelMetadata> = { + "kimi/k2": { vision: true }, + "kimi/k1.5": { vision: true }, + "umans/glm-5.2": { vision: false }, + "openai/gpt-4": { contextWindow: 128000 }, + }; + + it("includes the Auto option first", () => { + const opts = compactionModelOptions([], {}); + expect(opts).toHaveLength(1); + expect(opts[0]?.auto).toBe(true); + expect(opts[0]?.value).toBe(AUTO_COMPACTION_MODEL); + }); + + it("includes only vision-capable models, in catalog order", () => { + const models = ["umans/glm-5.2", "kimi/k2", "openai/gpt-4", "kimi/k1.5"]; + const opts = compactionModelOptions(models, modelInfo); + expect(opts.map((o) => o.label)).toEqual(["Auto (server-selected)", "kimi/k2", "kimi/k1.5"]); + }); + + it("excludes non-vision models even with metadata present", () => { + const opts = compactionModelOptions(["umans/glm-5.2"], modelInfo); + expect(opts).toHaveLength(1); // only Auto + }); +}); + +describe("compactionModel value round-trip", () => { + it("selectedCompactionValue maps null to the auto sentinel", () => { + expect(selectedCompactionValue(null)).toBe(AUTO_COMPACTION_MODEL); + }); + + it("selectedCompactionValue maps a model name to itself", () => { + expect(selectedCompactionValue("kimi/k2")).toBe("kimi/k2"); + }); + + it("compactionModelFromValue maps the auto sentinel back to null", () => { + expect(compactionModelFromValue(AUTO_COMPACTION_MODEL)).toBeNull(); + }); + + it("compactionModelFromValue maps a model name to itself", () => { + expect(compactionModelFromValue("kimi/k2")).toBe("kimi/k2"); + }); +}); + +describe("compactionModelChanged", () => { + it("is true when the value maps to a different model", () => { + expect(compactionModelChanged("kimi/k2", null)).toBe(true); + expect(compactionModelChanged(AUTO_COMPACTION_MODEL, "kimi/k2")).toBe(true); + }); + + it("is false when equal (null vs auto, or same model)", () => { + expect(compactionModelChanged(AUTO_COMPACTION_MODEL, null)).toBe(false); + expect(compactionModelChanged("kimi/k2", "kimi/k2")).toBe(false); + }); +}); + +describe("imageLimitLabel", () => { + it("labels null as loading", () => { + expect(imageLimitLabel(null)).toBe("Loading…"); + }); + + it("labels 0 as disabled", () => { + expect(imageLimitLabel(0)).toBe("0 (compaction disabled)"); + }); + + it("labels the default with (default)", () => { + expect(imageLimitLabel(DEFAULT_IMAGE_LIMIT)).toBe(`${DEFAULT_IMAGE_LIMIT} (default)`); + }); + + it("labels other values plainly", () => { + expect(imageLimitLabel(7)).toBe("7"); + }); +}); diff --git a/src/features/vision/logic/view-model.ts b/src/features/vision/logic/view-model.ts new file mode 100644 index 0000000..97a7093 --- /dev/null +++ b/src/features/vision/logic/view-model.ts @@ -0,0 +1,189 @@ +import type { ModelMetadata } from "@dispatch/transport-contract"; +import { isVisionModel } from "../../chat"; + +/** + * Pure core for the vision settings feature — zero DOM, zero effects, zero Svelte. + * + * The global vision configuration (`GET`/`PUT /settings/vision`) controls image + * compaction: how many native images a vision model keeps per turn before the + * oldest are transcribed to text (`imageLimit`), and which vision model does the + * transcribing (`compactionModel`, null = auto). This module is the view-model + * seam: typed settings, parse/validate, dirty-check, network normalization, and + * the vision-capable model option list. The composition root adapts the store's + * HTTP calls to the injected ports. + */ + +// ── Types (owned locally — consumer-defines-port; the contract shapes are a +// plain REST surface not in a shared contract package version bump, mirroring +// the heartbeat/mcp pattern). If the backend promotes these to a shared +// package, swap the local types for the imports + re-mirror. ────────────── + +/** The global vision settings (mirrors `VisionSettingsResponse`). */ +export interface VisionSettings { + /** Max native images per turn (default 10); 0 disables compaction. */ + readonly imageLimit: number; + /** Which model transcribes old images (`<key>/<model>`), or null = auto. */ + readonly compactionModel: string | null; +} + +/** A partial update (mirrors `SetVisionSettingsRequest`). */ +export interface VisionSettingsPatch { + readonly imageLimit?: number; + readonly compactionModel?: string | null; +} + +// ── Injected ports (consumer-defines-port). ─────────────────────────────────── + +/** Outcome of loading the vision settings. */ +export type LoadVisionSettingsResult = + | { readonly ok: true; readonly settings: VisionSettings } + | { readonly ok: false; readonly error: string }; + +/** Outcome of saving a partial vision-settings update. */ +export type SaveVisionSettingsResult = + | { readonly ok: true; readonly settings: VisionSettings } + | { readonly ok: false; readonly error: string }; + +export type LoadVisionSettings = () => Promise<LoadVisionSettingsResult>; +export type SaveVisionSettings = (patch: VisionSettingsPatch) => Promise<SaveVisionSettingsResult>; + +// ── Constants ──────────────────────────────────────────────────────────────── + +/** The backend's default image limit when none is persisted. */ +export const DEFAULT_IMAGE_LIMIT = 10; +/** Upper bound for the image limit input (defensive — the backend owns real clamping). */ +export const MAX_IMAGE_LIMIT = 1000; + +// ── Network normalization (pure; coerces untyped JSON at the seam). ────────── + +/** + * Coerce an untyped JSON body (from `GET /settings/vision`) into a valid + * `VisionSettings`. A malformed/partial response can never crash the renderer: + * `imageLimit` falls back to the default; `compactionModel` coerces to null. + */ +export function normalizeVisionSettings(body: unknown): VisionSettings { + if (body === null || typeof body !== "object") { + return { imageLimit: DEFAULT_IMAGE_LIMIT, compactionModel: null }; + } + const raw = body as Record<string, unknown>; + const limitRaw = raw.imageLimit; + const imageLimit = + typeof limitRaw === "number" && Number.isFinite(limitRaw) && limitRaw >= 0 + ? Math.floor(limitRaw) + : DEFAULT_IMAGE_LIMIT; + const modelRaw = raw.compactionModel; + const compactionModel = typeof modelRaw === "string" && modelRaw.length > 0 ? modelRaw : null; + return { imageLimit, compactionModel }; +} + +// ── imageLimit parse / validate ─────────────────────────────────────────────── + +/** Result of parsing a typed image-limit string. */ +export type ImageLimitParse = + | { readonly ok: true; readonly value: number } + | { readonly ok: false; readonly error: string }; + +/** + * Parse a typed image-limit string into a non-negative integer (floored, + * clamped to [0, MAX_IMAGE_LIMIT]). Empty or non-numeric input is an ERROR + * (so the UI can disable submit + message), NOT a silent default. + */ +export function parseImageLimit(raw: string): ImageLimitParse { + const trimmed = raw.trim(); + if (trimmed.length === 0) { + return { ok: false, error: "Enter a number." }; + } + const n = Number(trimmed); + if (!Number.isFinite(n) || n < 0) { + return { ok: false, error: "Must be 0 or a positive number." }; + } + const floored = Math.floor(n); + if (floored > MAX_IMAGE_LIMIT) { + return { ok: false, error: `Must be at most ${MAX_IMAGE_LIMIT}.` }; + } + return { ok: true, value: floored }; +} + +/** + * Whether saving `typed` would change the `current` image limit. A no-op save + * (empty/invalid, or equal) should be disabled. This is the dirty-check for the + * input — it must NOT mutate or clamp, only compare. + */ +export function imageLimitChanged(typed: string, current: number): boolean { + const parsed = parseImageLimit(typed); + if (!parsed.ok) return false; + return parsed.value !== current; +} + +// ── compactionModel option list ─────────────────────────────────────────────── + +/** + * The sentinel value for the "Auto" option (null compactionModel — the server + * auto-selects a vision model). Used as the `<option value>` for the auto row. + */ +export const AUTO_COMPACTION_MODEL = "__auto__"; + +/** A selectable compaction-model option. */ +export interface CompactionModelOption { + /** The value to send (`<key>/<model>`, or `AUTO_COMPACTION_MODEL` for auto). */ + readonly value: string; + /** The human-readable label. */ + readonly label: string; + /** Whether this is the "Auto" sentinel. */ + readonly auto: boolean; +} + +/** + * Build the compaction-model dropdown options: the "Auto" entry (null) plus + * every vision-capable model from the catalog (those with + * `modelInfo[name].vision === true`), in catalog order. Non-vision models are + * excluded — they cannot transcribe images. + */ +export function compactionModelOptions( + models: readonly string[], + modelInfo: Readonly<Record<string, ModelMetadata>>, +): CompactionModelOption[] { + const options: CompactionModelOption[] = [ + { value: AUTO_COMPACTION_MODEL, label: "Auto (server-selected)", auto: true }, + ]; + for (const name of models) { + if (isVisionModel(modelInfo, name)) { + options.push({ value: name, label: name, auto: false }); + } + } + return options; +} + +/** + * The `<option value>` to mark selected for the current `compactionModel`: + * `AUTO_COMPACTION_MODEL` when null (auto), else the model name itself. + */ +export function selectedCompactionValue(compactionModel: string | null): string { + return compactionModel ?? AUTO_COMPACTION_MODEL; +} + +/** + * Convert a selected `<option value>` back into a `VisionSettingsPatch` + * `compactionModel` (the auto sentinel → null). Returns the patch alone so the + * caller can merge it with other fields. + */ +export function compactionModelFromValue(value: string): string | null { + return value === AUTO_COMPACTION_MODEL ? null : value; +} + +/** + * Whether choosing `value` would change the current `compactionModel`. + */ +export function compactionModelChanged(value: string, current: string | null): boolean { + return compactionModelFromValue(value) !== current; +} + +// ── Labels ──────────────────────────────────────────────────────────────────── + +/** A human-readable label for the current image limit (for the status line). */ +export function imageLimitLabel(imageLimit: number | null): string { + if (imageLimit === null) return "Loading…"; + if (imageLimit === 0) return "0 (compaction disabled)"; + if (imageLimit === DEFAULT_IMAGE_LIMIT) return `${imageLimit} (default)`; + return String(imageLimit); +} diff --git a/src/features/vision/ui/VisionSettingsView.svelte b/src/features/vision/ui/VisionSettingsView.svelte new file mode 100644 index 0000000..2b4ebba --- /dev/null +++ b/src/features/vision/ui/VisionSettingsView.svelte @@ -0,0 +1,192 @@ +<script lang="ts"> + import type { ModelMetadata } from "@dispatch/transport-contract"; + import { + compactionModelChanged, + compactionModelFromValue, + compactionModelOptions, + DEFAULT_IMAGE_LIMIT, + imageLimitChanged, + imageLimitLabel, + parseImageLimit, + selectedCompactionValue, + type LoadVisionSettings, + type SaveVisionSettings, + type VisionSettings, + } from "../logic/view-model"; + + let { + models, + modelInfo = {}, + load, + save, + }: { + /** The model catalog (`GET /models` `models`) — for the compaction-model dropdown. */ + models: readonly string[]; + /** Per-model metadata — to filter the dropdown to vision-capable models. */ + modelInfo?: Readonly<Record<string, ModelMetadata>>; + /** Load the global vision settings (`GET /settings/vision`). */ + load: LoadVisionSettings; + /** Save a partial vision-settings update (`PUT /settings/vision`). */ + save: SaveVisionSettings; + } = $props(); + + let settings = $state<VisionSettings | null>(null); + let loadError = $state<string | null>(null); + + // imageLimit input state. + let imageLimitInput = $state(""); + let savingImageLimit = $state(false); + let imageLimitError = $state<string | null>(null); + let imageLimitSaved = $state(false); + + // compactionModel select state. + let compactionSaving = $state(false); + let compactionError = $state<string | null>(null); + let compactionSaved = $state(false); + + // Load on mount. + $effect(() => { + void refresh(); + }); + + async function refresh(): Promise<void> { + const result = await load(); + if (result.ok) { + settings = result.settings; + imageLimitInput = String(result.settings.imageLimit); + loadError = null; + imageLimitError = null; + imageLimitSaved = false; + compactionError = null; + compactionSaved = false; + } else { + loadError = result.error; + } + } + + const options = $derived(compactionModelOptions(models, modelInfo)); + const selectedCompaction = $derived( + settings ? selectedCompactionValue(settings.compactionModel) : selectedCompactionValue(null), + ); + const limitLabel = $derived(imageLimitLabel(settings?.imageLimit ?? null)); + + const canSaveImageLimit = $derived( + settings !== null && imageLimitChanged(imageLimitInput, settings.imageLimit), + ); + + async function handleSaveImageLimit(): Promise<void> { + if (settings === null || savingImageLimit) return; + const parsed = parseImageLimit(imageLimitInput); + if (!parsed.ok) { + imageLimitError = parsed.error; + imageLimitSaved = false; + return; + } + savingImageLimit = true; + imageLimitError = null; + imageLimitSaved = false; + const result = await save({ imageLimit: parsed.value }); + savingImageLimit = false; + if (result.ok) { + settings = result.settings; + imageLimitInput = String(result.settings.imageLimit); + imageLimitSaved = true; + } else { + imageLimitError = result.error; + } + } + + async function handleCompactionChange(e: Event): Promise<void> { + if (settings === null || compactionSaving) return; + const value = (e.currentTarget as HTMLSelectElement).value; + if (!compactionModelChanged(value, settings.compactionModel)) return; + compactionSaving = true; + compactionError = null; + compactionSaved = false; + const result = await save({ compactionModel: compactionModelFromValue(value) }); + compactionSaving = false; + if (result.ok) { + settings = result.settings; + compactionSaved = true; + } else { + compactionError = result.error; + } + } +</script> + +<div class="flex flex-col gap-3"> + {#if loadError} + <p class="text-xs text-error">{loadError}</p> + {/if} + + {#if settings === null && !loadError} + <p class="text-xs opacity-60">Loading vision settings…</p> + {:else if settings !== null} + <!-- imageLimit --> + <section class="flex flex-col gap-1"> + <span class="text-xs font-semibold uppercase opacity-60">Image limit</span> + <div class="flex items-center gap-2"> + <input + type="text" + inputmode="numeric" + class="input input-bordered input-sm w-24" + placeholder={String(DEFAULT_IMAGE_LIMIT)} + bind:value={imageLimitInput} + disabled={savingImageLimit} + aria-label="Image limit (max native images per turn)" + /> + <button + type="button" + class="btn btn-sm btn-outline" + disabled={!canSaveImageLimit || savingImageLimit} + onclick={handleSaveImageLimit} + > + {#if savingImageLimit} + <span class="loading loading-spinner loading-xs"></span> + Saving… + {:else} + Save + {/if} + </button> + </div> + <p class="text-xs opacity-50"> + Current: {limitLabel} + <br /> + Max native images per turn before the oldest are transcribed to text. + 0 disables compaction. Default is {DEFAULT_IMAGE_LIMIT}. + </p> + {#if imageLimitError} + <p class="text-xs text-error">{imageLimitError}</p> + {:else if imageLimitSaved} + <p class="text-xs text-success">Saved.</p> + {/if} + </section> + + <!-- compactionModel --> + <section class="flex flex-col gap-1"> + <span class="text-xs font-semibold uppercase opacity-60">Compaction model</span> + <select + class="select select-bordered select-sm w-full" + value={selectedCompaction} + disabled={compactionSaving} + onchange={handleCompactionChange} + aria-label="Compaction model (which vision model transcribes old images)" + > + {#each options as opt (opt.value)} + <option value={opt.value}>{opt.label}</option> + {/each} + </select> + {#if compactionSaving} + <p class="text-xs opacity-60">Saving…</p> + {/if} + <p class="text-xs opacity-50"> + The vision-capable model that transcribes old images to text. "Auto" lets the server choose. + </p> + {#if compactionError} + <p class="text-xs text-error">{compactionError}</p> + {:else if compactionSaved} + <p class="text-xs text-success">Saved.</p> + {/if} + </section> + {/if} +</div> diff --git a/src/features/vision/ui/VisionSettingsView.test.ts b/src/features/vision/ui/VisionSettingsView.test.ts new file mode 100644 index 0000000..48afc71 --- /dev/null +++ b/src/features/vision/ui/VisionSettingsView.test.ts @@ -0,0 +1,241 @@ +import { render, screen } from "@testing-library/svelte"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import type { + LoadVisionSettingsResult, + SaveVisionSettingsResult, + VisionSettings, +} from "../logic/view-model"; +import VisionSettingsView from "./VisionSettingsView.svelte"; + +const SETTINGS: VisionSettings = { imageLimit: 10, compactionModel: null }; + +function fakeLoad(settings: VisionSettings = SETTINGS): { + calls: number; + impl: () => Promise<LoadVisionSettingsResult>; +} { + let calls = 0; + return { + get calls() { + return calls; + }, + impl: async () => { + calls += 1; + return { ok: true, settings }; + }, + }; +} + +function fakeSaveOk(): { + patches: object[]; + impl: (patch: object) => Promise<SaveVisionSettingsResult>; +} { + const patches: object[] = []; + return { + get patches() { + return patches; + }, + impl: async (patch) => { + patches.push(patch); + // Merge into the current settings to simulate the server echo. + const next: VisionSettings = { + imageLimit: + "imageLimit" in patch ? (patch as VisionSettings).imageLimit : SETTINGS.imageLimit, + compactionModel: + "compactionModel" in patch + ? (patch as VisionSettings).compactionModel + : SETTINGS.compactionModel, + }; + return { ok: true, settings: next }; + }, + }; +} + +describe("VisionSettingsView", () => { + it("loads settings on mount and seeds the imageLimit input", async () => { + const load = fakeLoad({ imageLimit: 7, compactionModel: "kimi/k2" }); + render(VisionSettingsView, { + props: { + models: ["kimi/k2"], + modelInfo: { "kimi/k2": { vision: true } }, + load: load.impl, + save: fakeSaveOk().impl, + }, + }); + + await vi.waitFor(() => { + expect(screen.getByLabelText(/Image limit/)).toHaveValue("7"); + }); + // "Auto" is selected (compactionModel was kimi/k2 here actually) + expect(screen.getByLabelText(/Compaction model/)).toHaveValue("kimi/k2"); + }); + + it("disables Save until the imageLimit input differs", async () => { + const load = fakeLoad(); + const save = fakeSaveOk(); + const user = userEvent.setup(); + render(VisionSettingsView, { + props: { models: [], modelInfo: {}, load: load.impl, save: save.impl }, + }); + + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Save" })).toBeDisabled(); + }); + + const input = screen.getByLabelText(/Image limit/); + await user.clear(input); + await user.type(input, "5"); + expect(screen.getByRole("button", { name: "Save" })).toBeEnabled(); + }); + + it("saves the imageLimit on click and confirms", async () => { + const load = fakeLoad(); + const save = fakeSaveOk(); + const user = userEvent.setup(); + render(VisionSettingsView, { + props: { models: [], modelInfo: {}, load: load.impl, save: save.impl }, + }); + + await vi.waitFor(() => { + expect(screen.getByLabelText(/Image limit/)).toHaveValue("10"); + }); + + const input = screen.getByLabelText(/Image limit/); + await user.clear(input); + await user.type(input, "3"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await vi.waitFor(() => { + expect(save.patches).toEqual([{ imageLimit: 3 }]); + }); + expect(screen.getByText(/Saved/i)).toBeInTheDocument(); + }); + + it("shows an error for a non-numeric imageLimit on save", async () => { + const load = fakeLoad(); + const save = fakeSaveOk(); + const user = userEvent.setup(); + render(VisionSettingsView, { + props: { models: [], modelInfo: {}, load: load.impl, save: save.impl }, + }); + + await vi.waitFor(() => { + expect(screen.getByLabelText(/Image limit/)).toHaveValue("10"); + }); + + const input = screen.getByLabelText(/Image limit/); + await user.clear(input); + await user.type(input, "abc"); + // Save is disabled for invalid input, so no save fires; the error surfaces + // only on a submit attempt — but the button is disabled, so just assert that. + expect(screen.getByRole("button", { name: "Save" })).toBeDisabled(); + expect(save.patches).toEqual([]); + }); + + it("renders the compaction-model dropdown with Auto + vision-capable models", async () => { + const load = fakeLoad(); + render(VisionSettingsView, { + props: { + models: ["kimi/k2", "umans/glm-5.2", "kimi/k1.5"], + modelInfo: { + "kimi/k2": { vision: true }, + "kimi/k1.5": { vision: true }, + "umans/glm-5.2": { vision: false }, + }, + load: load.impl, + save: fakeSaveOk().impl, + }, + }); + + await vi.waitFor(() => { + expect(screen.getByLabelText(/Compaction model/)).toBeInTheDocument(); + }); + const select = screen.getByLabelText(/Compaction model/) as HTMLSelectElement; + const optionTexts = Array.from(select.options).map((o) => o.textContent ?? ""); + expect(optionTexts).toEqual(["Auto (server-selected)", "kimi/k2", "kimi/k1.5"]); + // Non-vision glm-5.2 is excluded. + expect(optionTexts.some((t) => t.includes("glm-5.2"))).toBe(false); + }); + + it("saves the compactionModel on change (Auto → a vision model)", async () => { + const load = fakeLoad({ imageLimit: 10, compactionModel: null }); + const save = fakeSaveOk(); + const user = userEvent.setup(); + render(VisionSettingsView, { + props: { + models: ["kimi/k2"], + modelInfo: { "kimi/k2": { vision: true } }, + load: load.impl, + save: save.impl, + }, + }); + + await vi.waitFor(() => { + expect(screen.getByLabelText(/Compaction model/)).toBeInTheDocument(); + }); + + await user.selectOptions(screen.getByLabelText(/Compaction model/), "kimi/k2"); + + await vi.waitFor(() => { + expect(save.patches).toEqual([{ compactionModel: "kimi/k2" }]); + }); + expect(screen.getByText(/Saved/i)).toBeInTheDocument(); + }); + + it("saves null (Auto) when the auto option is chosen", async () => { + const load = fakeLoad({ imageLimit: 10, compactionModel: "kimi/k2" }); + const save = fakeSaveOk(); + const user = userEvent.setup(); + render(VisionSettingsView, { + props: { + models: ["kimi/k2"], + modelInfo: { "kimi/k2": { vision: true } }, + load: load.impl, + save: save.impl, + }, + }); + + await vi.waitFor(() => { + expect(screen.getByLabelText(/Compaction model/)).toHaveValue("kimi/k2"); + }); + + await user.selectOptions(screen.getByLabelText(/Compaction model/), "__auto__"); + + await vi.waitFor(() => { + expect(save.patches).toEqual([{ compactionModel: null }]); + }); + }); + + it("surfaces a load error", async () => { + const load = vi.fn(async () => ({ ok: false, error: "vision unavailable" }) as const); + render(VisionSettingsView, { + props: { models: [], modelInfo: {}, load, save: fakeSaveOk().impl }, + }); + + await vi.waitFor(() => { + expect(screen.getByText("vision unavailable")).toBeInTheDocument(); + }); + }); + + it("surfaces a save error", async () => { + const load = fakeLoad(); + const save = vi.fn(async () => ({ ok: false, error: "boom" }) as const); + const user = userEvent.setup(); + render(VisionSettingsView, { + props: { models: [], modelInfo: {}, load: load.impl, save }, + }); + + await vi.waitFor(() => { + expect(screen.getByLabelText(/Image limit/)).toHaveValue("10"); + }); + + const input = screen.getByLabelText(/Image limit/); + await user.clear(input); + await user.type(input, "3"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await vi.waitFor(() => { + expect(screen.getByText("boom")).toBeInTheDocument(); + }); + }); +}); |
