summaryrefslogtreecommitdiffhomepage
path: root/.dispatch/transport-contract.reference.md
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-27 20:49:43 +0900
committerAdam Malczewski <[email protected]>2026-06-27 20:49:43 +0900
commita59200e786f7d97d7ba5b9cd2bee9ffef531dac2 (patch)
tree92088287487e34a9fee37f89325961110b1d3442 /.dispatch/transport-contract.reference.md
parenta9ca756de8cd023c0f2cb9954f344fff11146bc2 (diff)
parentb70ae547fdcb8c1794981957485537dc21a8b5fd (diff)
downloaddispatch-web-a59200e786f7d97d7ba5b9cd2bee9ffef531dac2.tar.gz
dispatch-web-a59200e786f7d97d7ba5b9cd2bee9ffef531dac2.zip
Merge branch 'feature/vision-handoff' into dev
# Conflicts: # .dispatch/transport-contract.reference.md # backend-handoff.md # src/app/App.svelte # src/features/chat/ui/Composer.svelte
Diffstat (limited to '.dispatch/transport-contract.reference.md')
-rw-r--r--.dispatch/transport-contract.reference.md77
1 files changed, 77 insertions, 0 deletions
diff --git a/.dispatch/transport-contract.reference.md b/.dispatch/transport-contract.reference.md
index 3522cbb..6a5a6b3 100644
--- a/.dispatch/transport-contract.reference.md
+++ b/.dispatch/transport-contract.reference.md
@@ -15,6 +15,32 @@
> `concurrency` extension tracks/limits in-flight token-generating requests per provider with
> oldest-agent-first queueing; when it isn't loaded the list + status endpoints return empty arrays and
> the single/PUT/DELETE return `503`. See `backend-handoff.md` §2j.
+> **2026-06-26 delta (vision handoff — ADDITIVE, NO version bump):** adds the vision/image surface.
+> `ChatRequest` (+ `ChatSendMessage`/`QueueRequest`) gains an optional `images?: readonly ImageInput[]`
+> (each entry: `{ url, mimeType? }` — a base64 data URL or `http(s)://` URL; validated non-array/no-url/
+> empty-url → 400, empty array treated as absent). `ModelMetadata` gains `vision?: boolean` (true when the
+> model natively accepts images; absent → the server's vision handoff transcribes images to text before the
+> model sees them). `ImageChunk`/`ImageInput` are `@dispatch/wire` types (re-exported here).
+>
+> **2026-06-26 update (consult_vision + vision settings — ADDITIVE, NO version bump):** the `read_image`
+> tool is REPLACED by `consult_vision` (`{ question: string, imageIds?: number[], path?: string }`) — it
+> opens a NEW conversation tab with a vision-capable model, attaches the image + question, and returns the
+> vision model's answer (rendered like any tool call/result). Non-vision models now get NUMBERED
+> PLACEHOLDERS (`[Image N attached — call consult_vision with imageIds=[N] and a specific question to
+> analyze it]`) instead of auto-transcriptions — these are regular `text` chunks (render as-is). Image
+> compaction transcribes the oldest images past `imageLimit` to `[Compacted image]: <description>` text
+> chunks (also regular `text` — render as-is; the persisted `image` chunk stays for rendering). NEW global
+> vision settings API: `GET /settings/vision` → `VisionSettingsResponse` (`{ imageLimit, compactionModel }`),
+> `PUT /settings/vision` ← `SetVisionSettingsRequest` (partial: `imageLimit?` non-negative int, 0 = disable
+> compaction; `compactionModel?` `<key>/<model>` or null = auto). See `backend-handoff.md` §2j.
+>
+> **2026-06-26 update (image storage — NO type change, behavior only):** persisted `ImageChunk.url`s are now
+> compact relative HTTP paths (`/images/<conversationId>/<uuid>.png`) served by the new
+> `GET /images/:conversationId/:imageId` endpoint (raw image bytes + correct Content-Type) — NOT base64 data
+> URLs (images are stored on disk under tmp, not in the SQLite store). `ChatRequest.images` (`ImageInput.url`)
+> is UNCHANGED — clients still send data URLs; the backend saves them to tmp and returns compact paths in
+> the persisted chunks. A client resolves a relative `url` against its API base (`resolveImageUrl`); the
+> optimistic echo's data URL and any absolute URL pass through. See `backend-handoff.md` §2j.
>
> **2026-06-25 delta (SSH handoff #2 — ADDITIVE to `[email protected]`, NO version bump):** adds the
> computer HTTP API types: `ComputerListResponse` (`GET /computers`), `ComputerResponse` (`GET /computers/:alias`),
@@ -67,8 +93,11 @@
import type { SurfaceClientMessage, SurfaceServerMessage } from "@dispatch/ui-contract";
import type {
AgentEvent,
+ Computer,
+ ComputerEntry,
ConversationMeta,
ConversationStatus,
+ ImageInput,
QueuedMessage,
ReasoningEffort,
StoredChunk,
@@ -80,8 +109,12 @@ import type {
export type {
AgentEvent,
CompactionResult,
+ Computer,
+ ComputerEntry,
ConversationMeta,
ConversationStatus,
+ ImageChunk,
+ ImageInput,
QueuedMessage,
ReasoningEffort,
StepMetrics,
@@ -109,6 +142,21 @@ export interface ChatRequest {
readonly message: string;
/**
+ * Images attached to this turn (e.g. a user-pasted screenshot). Each entry's
+ * `url` is a base64 data URL (`data:image/…;base64,…`) or an `http(s)://`
+ * URL. The server converts these to `image` chunks on the persisted user
+ * message. For a VISION-capable model (e.g. kimi), the images are passed
+ * through to the provider natively. For a NON-vision model (e.g. glm-5.2),
+ * the server's vision handoff transcribes each image to a text description
+ * (via a vision-capable model) and feeds that text instead — so a text-only
+ * model can still reason about the image's contents. Optional — omit for a
+ * text-only turn (backward compatible). Validation: non-array `images` →
+ * 400; an image without `url` → 400; empty `url` → 400. An empty array is
+ * accepted and treated as absent.
+ */
+ readonly images?: readonly ImageInput[];
+
+ /**
* The model to use, as a model name in `<credentialName>/<model>` form — one
* of the exact strings returned by `GET /models`. Omit to use the server's
* default credential + model.
@@ -165,6 +213,14 @@ export interface ModelsResponse {
/** Per-model metadata returned alongside the model catalog. */
export interface ModelMetadata {
readonly contextWindow?: number;
+ /**
+ * Whether this model can natively accept image input (vision/multimodal).
+ * When `true`, image chunks in a user message are passed through to the
+ * provider. When `false`/absent, the server's vision handoff transcribes
+ * images to text before the model sees them. A client may use this to show
+ * a vision badge in the model picker. Optional — absent when unknown.
+ */
+ readonly vision?: boolean;
}
/**
@@ -428,6 +484,27 @@ export interface SystemPromptVariablesResponse {
readonly variables: readonly SystemPromptVariable[];
}
+// ─── Vision settings (global) ───────────────────────────────────────────────
+
+/**
+ * Response of `GET /settings/vision` — the global vision configuration shared
+ * across all conversations and vision models.
+ */
+export interface VisionSettingsResponse {
+ /** Max native images per turn (default 10); 0 disables image compaction. */
+ readonly imageLimit: number;
+ /** Which model transcribes old images (null = auto-select a vision model). */
+ readonly compactionModel: string | null;
+}
+
+/** Body of `PUT /settings/vision` — a partial update. */
+export interface SetVisionSettingsRequest {
+ /** Non-negative integer (0 = disable compaction). */
+ readonly imageLimit?: number;
+ /** A model name (`<key>/<model>`) or null (auto). */
+ readonly compactionModel?: string | null;
+}
+
// ─── Message queue (steering) ─────────────────────────────────────────────────
/**