From fa0bd9c0e433b1abddc814b48a358c94954c7d36 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 19:15:04 +0900 Subject: feat(vision): consult_vision tool + vision settings API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend vision update (additive to wire@0.12.0 / transport-contract@0.22.0, no version bump). Contracts mirrored (.dispatch/transport-contract.reference.md): - VisionSettingsResponse + SetVisionSettingsRequest (GET/PUT /settings/vision). - Delta note: read_image -> consult_vision; numbered placeholders; compaction. Tool rendering (ChatView): - read_image test -> consult_vision (rendering is generic by toolName). - +2 tests: numbered-placeholder text chunk + [Compacted image] text chunk (both regular text chunks, render as-is — no special handling). New vision feature library (src/features/vision/): - logic/view-model.ts (32 tests): VisionSettings/VisionSettingsPatch types (consumer-defines-port), LoadVisionSettings/SaveVisionSettings ports + results, normalizeVisionSettings (network-seam coercion), parseImageLimit/ imageLimitChanged, compactionModelOptions (vision-capable models via chat's public isVisionModel + Auto sentinel), round-trip helpers, imageLimitLabel. - ui/VisionSettingsView.svelte (9 tests): imageLimit input + Save, compactionModel dropdown (Auto + vision-capable models), load-on-mount, save-on-change, error/saved feedback. - index.ts. Cross-unit seam: isVisionModel added to features/chat public index.ts (additive); imported through the public surface, not internals. Store wiring (src/app/store.svelte.ts): - visionSettings state + refreshVisionSettings (GET /settings/vision, normalized at the seam) + setVisionSettings (PUT, partial, returns merged) + VisionSettingsResult; seeded on boot; exposed on AppStore. +4 store tests. Mounted in App.svelte: new "Vision" sidebar view kind + VisionSettingsView in viewContent (not conversation-scoped); load/save adapters; visionManifest. Verification: svelte-check 0/0; vitest 948/948 (run twice, +47 since the prior vision commit); biome clean; vite build OK. See backend-handoff.md §2j. Not merged or pushed. --- src/features/chat/index.ts | 1 + 1 file changed, 1 insertion(+) (limited to 'src/features/chat/index.ts') diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts index ddb094d..8694691 100644 --- a/src/features/chat/index.ts +++ b/src/features/chat/index.ts @@ -6,6 +6,7 @@ export type { } from "../../core/chunks"; export { groupRenderedChunks, viewProviderRetry } from "../../core/chunks"; export type { TurnMetricsEntry } from "../../core/metrics"; +export { isVisionModel } from "./model-select"; export type { ChatTransport, HistorySync, HistoryWindow, MetricsSync } from "./ports"; export type { EffortOption, -- cgit v1.2.3 From f5dc22f7c14d6c0dd4bcedee5a85b21ecd294aed Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 20:20:02 +0900 Subject: feat(vision): resolve persisted image URLs against the API base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Images are now stored on disk under tmp (not SQLite) and served via GET /images/:conversationId/:imageId. Persisted ImageChunk.url is a compact relative HTTP path (/images//.png) instead of a base64 data URL. No wire/transport-contract type change (behavior only) — re-mirrored the delta notes. - New pure resolveImageUrl(url, apiBase) helper (core/chunks/image-url.ts, +8 tests): data/absolute URLs pass through; relative paths are prepended with the API base (no double slash; empty base -> root-relative). Exported from core/chunks + re-exported from features/chat. - ChatView: new apiBaseUrl prop; uses resolveImageUrl. The optimistic echo's data URL passes through; persisted relative paths resolve against the base. +3 tests. - AppStore exposes httpBase (getter); App.svelte passes apiBaseUrl into ChatView and the heartbeat RunModal (also renders image chunks). Verification: svelte-check 0/0; vitest 959/959 (run twice, +11); biome clean; vite build OK. See backend-handoff.md §2j-update-2. Not merged or pushed. --- .dispatch/transport-contract.reference.md | 8 +++++ .dispatch/wire.reference.md | 28 ++++++++++++---- GLOSSARY.md | 2 +- backend-handoff.md | 53 +++++++++++++++++++++++++++---- src/app/App.svelte | 2 ++ src/app/store.svelte.ts | 9 ++++++ src/core/chunks/image-url.test.ts | 46 +++++++++++++++++++++++++++ src/core/chunks/image-url.ts | 35 ++++++++++++++++++++ src/core/chunks/index.ts | 1 + src/features/chat/index.ts | 2 +- src/features/chat/ui.test.ts | 52 ++++++++++++++++++++++++++++++ src/features/chat/ui/ChatView.svelte | 19 ++++++++--- src/features/heartbeat/ui/RunModal.svelte | 7 ++++ 13 files changed, 245 insertions(+), 19 deletions(-) create mode 100644 src/core/chunks/image-url.test.ts create mode 100644 src/core/chunks/image-url.ts (limited to 'src/features/chat/index.ts') diff --git a/.dispatch/transport-contract.reference.md b/.dispatch/transport-contract.reference.md index 047f050..182279a 100644 --- a/.dispatch/transport-contract.reference.md +++ b/.dispatch/transport-contract.reference.md @@ -27,6 +27,14 @@ > `PUT /settings/vision` ← `SetVisionSettingsRequest` (partial: `imageLimit?` non-negative int, 0 = disable > compaction; `compactionModel?` `/` or null = auto). See `backend-handoff.md` §2j. > +> **2026-06-26 update (image storage — NO type change, behavior only):** persisted `ImageChunk.url`s are now +> compact relative HTTP paths (`/images//.png`) served by the new +> `GET /images/:conversationId/:imageId` endpoint (raw image bytes + correct Content-Type) — NOT base64 data +> URLs (images are stored on disk under tmp, not in the SQLite store). `ChatRequest.images` (`ImageInput.url`) +> is UNCHANGED — clients still send data URLs; the backend saves them to tmp and returns compact paths in +> the persisted chunks. A client resolves a relative `url` against its API base (`resolveImageUrl`); the +> optimistic echo's data URL and any absolute URL pass through. See `backend-handoff.md` §2j. +> > **2026-06-25 delta (SSH handoff #2 — ADDITIVE to `transport-contract@0.22.0`, NO version bump):** adds the > computer HTTP API types: `ComputerListResponse` (`GET /computers`), `ComputerResponse` (`GET /computers/:alias`), > `ComputerStatusResponse` (`GET /computers/:alias/status`), `TestComputerResponse` (`POST /computers/:alias/test`), diff --git a/.dispatch/wire.reference.md b/.dispatch/wire.reference.md index c40fe81..75adac5 100644 --- a/.dispatch/wire.reference.md +++ b/.dispatch/wire.reference.md @@ -14,6 +14,15 @@ > — the orchestrator's vision handoff transcribes each to a text description (persisted as a separate > `text` chunk in the SAME user message). See `backend-handoff.md` §2j. > +> **2026-06-26 update (image storage — NO type change, behavior only):** `ImageChunk.url` for PERSISTED +> chunks is now a compact relative HTTP path (`/images//.png`) served by the backend's +> new `GET /images/:conversationId/:imageId` endpoint (raw bytes + correct Content-Type), NOT a base64 data +> URL — images are stored on disk under tmp, not in the SQLite conversation store (keeps payloads small). +> `ImageInput.url` (what a client SENDS on `ChatRequest.images`) is UNCHANGED — still a data URL or +> `http(s)://` URL; the backend saves it to tmp and returns the compact path in the persisted chunk. A client +> resolves a relative `url` against its API base (`resolveImageUrl`); a data URL (the optimistic echo) or an +> absolute URL passes through unchanged. See `backend-handoff.md` §2j. +> > **2026-06-23 delta (workspaces handoff — package bumped `0.11.0` → `0.12.0`, ADDITIVE):** adds > `Workspace` + `WorkspaceEntry` (a list entry with a conversation count) and a required > `workspaceId: string` on `ConversationMeta` (`"default"` for legacy/unspecified conversations). A @@ -156,12 +165,17 @@ export interface SystemChunk { /** * An image attached to a message (e.g. a user-pasted screenshot or pasted * photo). Carries a `url` that is EITHER a base64 data URL - * (`data:image/png;base64,…`) OR an `http(s)://` URL. Vision-capable models - * receive it natively (the provider serializes it to its image-content - * format); non-vision models never see it directly — the orchestrator's - * **vision handoff** transcribes it to a text description (via a - * vision-capable model) and feeds that text instead, so a text-only model can - * still reason about the image's contents. + * (`data:image/png;base64,…`) OR an `http(s)://` URL OR — for PERSISTED chunks + * (history/replay) — a compact relative HTTP path (`/images// + * .png`) served by the backend's `GET /images/:conversationId/:imageId` + * endpoint (images are stored on disk under tmp, NOT in the conversation store, + * to keep SQLite payloads small). A client resolves a relative path against its + * API base URL; a data URL (the optimistic echo / a pasted image) or an + * absolute URL is rendered as-is. Vision-capable models receive it natively + * (the provider serializes it to its image-content format); non-vision models + * never see it directly — the orchestrator's **vision handoff** transcribes it + * to a text description (via a vision-capable model) and feeds that text + * instead, so a text-only model can still reason about the image's contents. * * When a transcription was performed, it is persisted as a separate `text` * chunk alongside the `image` chunk in the SAME user message, so the @@ -170,7 +184,7 @@ export interface SystemChunk { */ export interface ImageChunk { readonly type: "image"; - /** Image source: a base64 data URL (`data:image/…;base64,…`) or an `http(s)://` URL. */ + /** Image source: a base64 data URL (`data:image/…;base64,…`), an `http(s)://` URL, or a compact relative path (`/images//.png`) for persisted chunks. */ readonly url: string; /** * Optional MIME type of the image (e.g. `"image/png"`). Inferred from the diff --git a/GLOSSARY.md b/GLOSSARY.md index 3973fe1..14b81d1 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -26,7 +26,7 @@ | **steering** | A user message injected into an in-flight turn at the tool-result boundary (drawn from the **message queue**): the model sees it alongside the tool results and may adjust course. Emitted on the chat stream as a `steering` `AgentEvent` (`TurnSteeringEvent`); the queue surface clears on drain (move, don't duplicate). If the turn ends with a non-empty queue (no tool call fired), the queue carries into a NEW turn as its opening prompt (no `steering` event). `wire@0.8.0`. | mid-turn injection, course correction, interruption | | **computer** | A remote SSH target discovered from the system's `~/.ssh/config` — a read-only VIEW, NOT an editable entity (no CRUD store; to add one the user edits `~/.ssh/config`). Backend-canonical (`wire@0.12.0`, additive). On the wire as `Computer` (`{ alias, hostName, port, user, identityFile, knownHost }`) + `ComputerEntry extends Computer` (adds `usageCount`, for `GET /computers`). `alias` IS the **computerId** — the string persisted per-conversation / per-workspace (the computer analog of `cwd`). Resolution is SERVER-owned (never re-implement): per-conversation `computerId` → `workspace.defaultComputerId` → `null`/local. USER-facing only: a tool-execution target forwarded to tools, NEVER part of the model prompt (does not affect prompt caching); the agent never sees it. HTTP API (`GET /computers`, `GET`/`PUT /conversations/:id/computer`, `PUT /workspaces/:id/default-computer`, `GET /computers/:alias/status`, `POST /computers/:alias/test`) consumed in handoff #2. | ssh host, remote host, server, connection target | | **computerId** | The string id of a **computer** — an SSH config `Host` alias users select. Persisted per-conversation and per-workspace (the computer analog of `cwd`/`workspaceId`). `null` means local (no SSH; today's behavior). On `Workspace` as the REQUIRED `defaultComputerId: string \| null` (null = local / no SSH; the computer analog of `defaultCwd`); per-conversation persistence via `GET`/`PUT`/`DELETE /conversations/:id/computer`. `chat.send` need not send it (resolved server-side from the persisted per-conversation value in the MVP). | ssh alias, host id, remote id | -| **image chunk** | An `ImageChunk` (`{ type: "image", url, mimeType? }`) — a NEW `Chunk` variant for an image attached to a message (a user-pasted screenshot/photo). `url` is a base64 data URL (`data:image/…;base64,…`) OR an `http(s)://` URL. Backend-canonical (`wire@0.12.0`, additive). A user message may be multi-chunk (`[text, image, image, …]` in order). On the wire as `ImageChunk` (persisted) + `ImageInput` (what `ChatRequest.images` carries; the orchestrator converts each into an `ImageChunk`). | picture, photo attachment, screenshot chunk | +| **image chunk** | An `ImageChunk` (`{ type: "image", url, mimeType? }`) — a `Chunk` variant for an image attached to a message (a user-pasted screenshot/photo). `url` is a base64 data URL (`data:image/…;base64,…`), an `http(s)://` URL, or — for PERSISTED chunks — a compact relative HTTP path (`/images//.png`) served by `GET /images/:conversationId/:imageId` (images are stored on disk under tmp, not in the SQLite store). Backend-canonical (`wire@0.12.0`, additive). A user message may be multi-chunk (`[text, image, image, …]` in order). On the wire as `ImageChunk` (persisted) + `ImageInput` (what `ChatRequest.images` carries — still a data URL; the orchestrator saves it to tmp and returns the compact path). A client resolves a relative `url` against its API base (`resolveImageUrl`); a data URL (the optimistic echo) or absolute URL passes through. | picture, photo attachment, screenshot chunk | | **vision** (capability) | Whether a model can natively accept image input (multimodal). On the wire as `ModelMetadata.vision?: boolean` (`GET /models` `modelInfo[name].vision`). `true` (e.g. any `kimi/*` model) → image chunks are passed through to the provider natively. Absent/`false` (e.g. `umans/glm-5.2`) → the server's **vision handoff** gives the model a numbered placeholder + the `consult_vision` tool. The FE shows a vision badge in the model picker; it does NOT decide handoff (server-owned). | multimodal, image support | | **vision handoff** | The server-owned mechanism by which a NON-vision model still reasons about a pasted image. A non-vision model gets a NUMBERED PLACEHOLDER text chunk alongside the persisted `image` chunk, then calls the `consult_vision` tool (which opens a NEW conversation tab with a vision-capable model, attaches the image + question, and returns the vision model's answer). The persisted user message keeps the original `image` chunk (the FE renders it) AND the placeholder text. When a vision-capable model has more than `imageLimit` images in history, image **compaction** transcribes the oldest to `[Compacted image]: ` text chunks (the `image` chunk stays for rendering). All these are regular `text` chunks — the FE renders them as-is. Distinct from a vision-capable model, which receives the image natively. | image transcription, vision relay | | **consult_vision** | A tool available to all models that defers image analysis to a vision-capable model: it opens a NEW conversation tab (with a vision model, e.g. Kimi), attaches the image (by `imageIds` from a pasted placeholder, or by `path` from disk) + a `question`, and returns the conversation id + the vision model's answer (suggesting the dispatch CLI for follow-ups). Replaces the former `read_image` tool. Rendered generically like any tool call/result (by `toolName`). | read_image (former), vision tool | diff --git a/backend-handoff.md b/backend-handoff.md index 9338a5a..5a476be 100644 --- a/backend-handoff.md +++ b/backend-handoff.md @@ -5,12 +5,11 @@ > **From:** dispatch-web orchestrator · **To:** `../backend` orchestrator · **Courier:** the user. > `lsp` does NOT span the repos (AGENTS.md § Backend seam) — every cross-repo ask flows through here. -_Last updated: 2026-06-26 (§2j UPDATED — consult_vision tool + vision settings API: `read_image` is REPLACED by -`consult_vision` (non-vision models now get numbered placeholder text chunks, not auto-transcriptions); NEW global -`GET`/`PUT /settings/vision` (`VisionSettingsResponse`/`SetVisionSettingsRequest` — `imageLimit` + `compactionModel`); -image compaction transcribes old images to `[Compacted image]: …` text chunks (all regular text — render as-is). New -`vision` feature library + "Vision" sidebar view (imageLimit input + compactionModel dropdown of vision-capable models + -"Auto"). typecheck 0/0, 948 tests green (+47), biome clean, build OK. §2i unchanged.)_ +_Last updated: 2026-06-26 (§2j UPDATED — Image storage: persisted `ImageChunk.url`s are now compact relative HTTP +paths (`/images//.png`) served by `GET /images/:conversationId/:imageId` (images stored on disk under tmp, +not SQLite). FE resolves relative urls against the API base via a new pure `resolveImageUrl` helper; the optimistic +echo's data URL passes through unchanged; `ChatRequest.images` (send) is unchanged. typecheck 0/0, 959 tests green +(+11), biome clean, build OK. §2i unchanged.)_ **FE is current on `ui-contract@0.2.0` / `transport-contract@0.22.0` / `wire@0.12.0`.** Open asks: **CR-9** (`system:os` should detect WSL + include Linux distro — backend behavior change, no contract bump). The SSH-divergence (§2d) is RESOLVED. @@ -854,6 +853,48 @@ reload; set compactionModel to a vision model → confirm the dropdown reflects non-vision model → confirm the `[Image N attached — call consult_vision…]` placeholder renders (not an auto-transcription); trigger a `consult_vision` tool call → confirm it renders like a tool. +### 2j-update-2. Image storage (tmp, not SQLite) → **CONSUMED ✅ (backend updated; FE built + verified)** + +A follow-up to §2j. The backend no longer persists images as base64 data URLs in the conversation store — +they are saved to a tmp directory and served via HTTP. The `ImageChunk.url` field's FORMAT changed (the +TYPE is unchanged — still `string`); `GET /images/:conversationId/:imageId` is a new endpoint serving raw +bytes + the correct Content-Type. **NO wire/transport-contract type change** (behavior only); re-mirrored +the delta notes in `.dispatch/wire.reference.md` + `.dispatch/transport-contract.reference.md`. + +**What changed (backend):** +- **BEFORE:** `ImageChunk.url` was a base64 data URL (`data:image/png;base64,…`). +- **NOW:** `ImageChunk.url` for PERSISTED chunks (history/replay) is a compact relative HTTP path + (`/images//.png`), served by `GET /images/:conversationId/:imageId` (raw image + bytes + Content-Type). Images live on disk under tmp, NOT in the SQLite conversation store (keeps + payloads small). +- **`ChatRequest.images` (`ImageInput.url`) is UNCHANGED** — the FE still sends data URLs; the backend + saves them to tmp and returns compact paths in the persisted chunks. +- **Optimistic echo:** the FE's provisional echo still uses the data URL it sent (immediate render); + when the persisted chunk arrives (via `loadSince`/`syncTail`/event stream), it carries the compact path + and the FE switches to rendering via the HTTP endpoint. +- The vision settings API, `consult_vision`, and image compaction are UNCHANGED (compaction resolves + compact URLs internally). + +**FE (DONE + verified):** +- **New pure helper `resolveImageUrl(url, apiBase)`** (`core/chunks/image-url.ts`, +8 tests, exported from + `core/chunks` + re-exported from `features/chat`): a `data:` URL or an `http(s)://` URL passes through + unchanged; a relative path (`/images/…`) is prepended with the API base (no double slash; an empty base + yields a root-relative path a browser resolves against its origin). Pure — zero DOM/Svelte. +- **`ChatView.svelte`:** new `apiBaseUrl` prop (default `""`); the `` now uses + `resolveImageUrl(rendered.chunk.url, apiBaseUrl)`. The optimistic echo's data URL + any absolute URL + pass through; persisted relative paths resolve against the base. +3 ChatView tests (relative-path + resolution, data-URL pass-through with a base set, root-relative when no base). +- **Store + wiring:** `httpBase` (the resolved HTTP API base) is now exposed as a getter on `AppStore`; + `App.svelte` passes `apiBaseUrl={store.httpBase}` to `ChatView` AND to the heartbeat `RunModal` (its + `ChatView` also renders image chunks — added an `apiBaseUrl` prop there, threaded from `App.svelte`). + +**Verification:** `svelte-check` 0/0; vitest **959/959** (run TWICE — no cross-test pollution; +11 since the +prior commit: 8 `resolveImageUrl`, 3 ChatView resolution), biome clean, `vite build` succeeds. +**Live probe NOT run** (backend not reachable headless). To confirm end-to-end: paste an image with a +vision model, send, confirm the image renders immediately (data-URL echo) AND continues to render after +the turn seals (the persisted compact-path `/images/…` resolved against `httpBase`); reload the +conversation → confirm the persisted image renders from the `/images/…` endpoint (not a data URL). + --- ## 3. Likely NEXT backend asks (heads-up, not yet requested) diff --git a/src/app/App.svelte b/src/app/App.svelte index f41d7ba..7d70f67 100644 --- a/src/app/App.svelte +++ b/src/app/App.svelte @@ -516,6 +516,7 @@ onShowEarlier={handleShowEarlier} thinkingKeyBase={store.activeChat.thinkingKeyBase} providerRetry={store.activeChat.providerRetry} + apiBaseUrl={store.httpBase} /> {/key} @@ -610,6 +611,7 @@ closeChat={closeRunChat} stopRun={stopHeartbeatRun} onClose={() => (heartbeatRun = null)} + apiBaseUrl={store.httpBase} /> {/key} {/if} diff --git a/src/app/store.svelte.ts b/src/app/store.svelte.ts index 554d5e0..8353820 100644 --- a/src/app/store.svelte.ts +++ b/src/app/store.svelte.ts @@ -157,6 +157,12 @@ export interface AppStore { readonly activeConversationId: string | null; /** The workspace currently in view (URL slug); tabs are filtered to it. */ readonly activeWorkspaceId: string; + /** + * The resolved HTTP API base URL (e.g. `http://localhost:24203`). Used to + * resolve relative image URLs served by the backend (`/images/…`) into + * absolute URLs for ``. + */ + readonly httpBase: string; readonly activeChat: ChatStore; readonly models: readonly string[]; /** Per-model metadata (contextWindow, etc.) from `GET /models`. */ @@ -1127,6 +1133,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { get activeWorkspaceId(): string { return activeWorkspaceId; }, + get httpBase(): string { + return httpBase; + }, setActiveWorkspace(workspaceId: string): void { activeWorkspaceId = workspaceId; // Reset to a fresh draft scoped to the new workspace so a new chat is diff --git a/src/core/chunks/image-url.test.ts b/src/core/chunks/image-url.test.ts new file mode 100644 index 0000000..8a79c09 --- /dev/null +++ b/src/core/chunks/image-url.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { resolveImageUrl } from "./image-url"; + +const BASE = "http://localhost:24203"; + +describe("resolveImageUrl", () => { + it("returns a data URL as-is (the optimistic echo / a pasted image)", () => { + const dataUrl = "data:image/png;base64,iVBORw0KGgo="; + expect(resolveImageUrl(dataUrl, BASE)).toBe(dataUrl); + }); + + it("returns an absolute http URL as-is", () => { + const abs = "https://example.com/img.png"; + expect(resolveImageUrl(abs, BASE)).toBe(abs); + }); + + it("prepends the api base to a relative /images/ path", () => { + expect(resolveImageUrl("/images/conv-123/abc-456.png", BASE)).toBe( + "http://localhost:24203/images/conv-123/abc-456.png", + ); + }); + + it("does not double the slash when the base has a trailing slash", () => { + expect(resolveImageUrl("/images/c/x.png", "http://localhost:24203/")).toBe( + "http://localhost:24203/images/c/x.png", + ); + }); + + it("adds a leading slash to a path-relative url without one", () => { + expect(resolveImageUrl("images/c/x.png", BASE)).toBe("http://localhost:24203/images/c/x.png"); + }); + + it("returns the relative path as-is when apiBase is empty (root-relative)", () => { + // A browser resolves a root-relative `/images/…` against the document origin. + expect(resolveImageUrl("/images/c/x.png", "")).toBe("/images/c/x.png"); + }); + + it("handles a relative path with an empty apiBase (path-relative without slash)", () => { + expect(resolveImageUrl("images/c/x.png", "")).toBe("/images/c/x.png"); + }); + + it("returns a data URL as-is even with an empty apiBase", () => { + const dataUrl = "data:image/jpeg;base64,AAAA"; + expect(resolveImageUrl(dataUrl, "")).toBe(dataUrl); + }); +}); diff --git a/src/core/chunks/image-url.ts b/src/core/chunks/image-url.ts new file mode 100644 index 0000000..e5ec756 --- /dev/null +++ b/src/core/chunks/image-url.ts @@ -0,0 +1,35 @@ +/** + * Resolve an `ImageChunk.url` into a renderable `` value. + * + * Persisted image chunks now carry a COMPACT HTTP path + * (`/images//.png`) served by the backend — NOT a base64 + * data URL (images are stored on disk under tmp, not in the conversation store, + * to keep SQLite payloads small). The optimistic echo (what the FE just sent in + * `ChatRequest.images`) still carries a data URL, and a chunk could also carry + * an absolute `http(s)://` URL, so the resolution is format-aware: + * + * - `data:` URL → returned as-is (the optimistic echo / a pasted data URL). + * - `http(s)://` → returned as-is (an absolute URL already). + * - anything else (a relative path like `/images/…`) → `apiBase` is prepended + * (with no double slash). An empty `apiBase` leaves a root-relative path, + * which a browser resolves against the document origin. + * + * Pure: input → output, zero DOM, zero Svelte. + * + * @param url The chunk's `url` (data URL, absolute, or relative path). + * @param apiBase The HTTP API base URL (e.g. `http://localhost:24203`). + */ +export function resolveImageUrl(url: string, apiBase: string): string { + if (url.startsWith("data:") || url.startsWith("http://") || url.startsWith("https://")) { + return url; + } + // A relative path (e.g. `/images/…`) — normalize to a leading slash and + // prepend the api base. With an empty base this yields a root-relative path + // (a browser resolves `/images/…` against the document origin). + const path = url.startsWith("/") ? url : `/${url}`; + if (apiBase.length === 0) return path; + // Join without a double slash: strip a trailing slash from the base, then + // append the (leading-slash) path verbatim. + const base = apiBase.endsWith("/") ? apiBase.slice(0, -1) : apiBase; + return `${base}${path}`; +} diff --git a/src/core/chunks/index.ts b/src/core/chunks/index.ts index eea2303..bdd6ce4 100644 --- a/src/core/chunks/index.ts +++ b/src/core/chunks/index.ts @@ -1,5 +1,6 @@ export type { RenderGroup, ToolBatchEntry } from "./groups"; export { groupRenderedChunks } from "./groups"; +export { resolveImageUrl } from "./image-url"; export { appendUserMessage, applyHistory, diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts index 8694691..773cb91 100644 --- a/src/features/chat/index.ts +++ b/src/features/chat/index.ts @@ -4,7 +4,7 @@ export type { RenderGroup, ToolBatchEntry, } from "../../core/chunks"; -export { groupRenderedChunks, viewProviderRetry } from "../../core/chunks"; +export { groupRenderedChunks, resolveImageUrl, viewProviderRetry } from "../../core/chunks"; export type { TurnMetricsEntry } from "../../core/metrics"; export { isVisionModel } from "./model-select"; export type { ChatTransport, HistorySync, HistoryWindow, MetricsSync } from "./ports"; diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index b0aa6f0..5f8067d 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -628,6 +628,58 @@ describe("ChatView", () => { expect(img?.getAttribute("loading")).toBe("lazy"); }); + it("resolves a persisted image chunk's relative url against apiBaseUrl", () => { + // Persisted image chunks now carry a compact relative path (`/images/…`) + // served by the backend — prepend the API base to render them. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url: "/images/conv-123/abc-456.png", mimeType: "image/png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe( + "http://localhost:24203/images/conv-123/abc-456.png", + ); + }); + + it("passes a data URL through unchanged even with apiBaseUrl set (optimistic echo)", () => { + // The optimistic echo (what the FE just sent) is still a data URL; it must + // NOT be mangled by the base-URL prepend. + const dataUrl = "data:image/png;base64,iVBOR="; + const chunks: RenderedChunk[] = [ + { seq: null, role: "user", chunk: { type: "image", url: dataUrl }, provisional: true }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe(dataUrl); + }); + + it("leaves a relative image url root-relative when apiBaseUrl is absent", () => { + // No apiBaseUrl → a browser resolves `/images/…` against the document origin. + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "user", + chunk: { type: "image", url: "/images/conv-1/x.png" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { props: { chunks } }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe("/images/conv-1/x.png"); + }); + it("renders a multi-chunk user message [text, image] and a transcription text", () => { // A non-vision model: the server persists the original image chunk AND a // transcription text chunk in the SAME user message — render both. diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte index 8081951..cd69071 100644 --- a/src/features/chat/ui/ChatView.svelte +++ b/src/features/chat/ui/ChatView.svelte @@ -1,6 +1,6 @@