summaryrefslogtreecommitdiffhomepage
path: root/packages/vision-handoff
diff options
context:
space:
mode:
Diffstat (limited to 'packages/vision-handoff')
-rw-r--r--packages/vision-handoff/package.json14
-rw-r--r--packages/vision-handoff/src/extension.ts198
-rw-r--r--packages/vision-handoff/src/index.ts21
-rw-r--r--packages/vision-handoff/src/pure.test.ts180
-rw-r--r--packages/vision-handoff/src/pure.ts156
-rw-r--r--packages/vision-handoff/src/service.test.ts375
-rw-r--r--packages/vision-handoff/src/service.ts689
-rw-r--r--packages/vision-handoff/src/tool.ts137
-rw-r--r--packages/vision-handoff/tsconfig.json12
9 files changed, 1782 insertions, 0 deletions
diff --git a/packages/vision-handoff/package.json b/packages/vision-handoff/package.json
new file mode 100644
index 0000000..b11f7ee
--- /dev/null
+++ b/packages/vision-handoff/package.json
@@ -0,0 +1,14 @@
+{
+ "name": "@dispatch/vision-handoff",
+ "version": "0.0.0",
+ "type": "module",
+ "private": true,
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "dependencies": {
+ "@dispatch/conversation-store": "workspace:*",
+ "@dispatch/credential-store": "workspace:*",
+ "@dispatch/kernel": "workspace:*",
+ "@dispatch/openai-stream": "workspace:*"
+ }
+}
diff --git a/packages/vision-handoff/src/extension.ts b/packages/vision-handoff/src/extension.ts
new file mode 100644
index 0000000..08fddca
--- /dev/null
+++ b/packages/vision-handoff/src/extension.ts
@@ -0,0 +1,198 @@
+/**
+ * vision-handoff extension — registers the universal vision handoff service +
+ * the `consult_vision` tool.
+ *
+ * The service performs provider-agnostic vision handoff: when a non-vision model
+ * (e.g. glm-5.2) receives an image, it replaces the image with a numbered
+ * placeholder and registers it for tool access. The `consult_vision` tool opens
+ * a NEW conversation tab with a vision-capable model (e.g. Kimi), attaches the
+ * image + the model's specific question, and returns the conversation ID + the
+ * vision model's answer. Follow-ups go through the dispatch CLI.
+ *
+ * Images are saved to a tmp directory (`/tmp/dispatch/images/<convId>/`) so the
+ * conversation store (SQLite) only holds a compact URL reference — not
+ * megabytes of base64. Tmp files are purged on reboot (ephemeral dir), after
+ * compaction (the transcription replaces the image), and on conversation close.
+ *
+ * Effects (filesystem, orchestrator) live here in the shell, injected into the
+ * service. The pure decisions live in `pure.ts`. No `console.*`; logging via
+ * `host.logger`.
+ */
+
+import { mkdir, readFile, rm, unlink, writeFile } from "node:fs/promises";
+import { extname, isAbsolute, join, resolve as pathResolve } from "node:path";
+import { conversationStoreHandle } from "@dispatch/conversation-store";
+import type { CredentialStore } from "@dispatch/credential-store";
+import { credentialStoreHandle } from "@dispatch/credential-store";
+import type { Extension, HostAPI, Manifest } from "@dispatch/kernel";
+import {
+ createVisionHandoffService,
+ orchestratorLocalHandle,
+ visionHandoffHandle,
+} from "./service.js";
+import { createConsultVisionTool } from "./tool.js";
+
+export const manifest: Manifest = {
+ id: "vision-handoff",
+ name: "Vision Handoff",
+ version: "0.0.0",
+ apiVersion: "^0.1.0",
+ trust: "bundled",
+ activation: "eager",
+ capabilities: { network: true },
+ contributes: { services: ["vision-handoff/service"], tools: ["consult_vision"] },
+};
+
+const IMAGE_DIR = process.env.DISPATCH_IMAGE_DIR ?? "/tmp/dispatch/images";
+
+/** MIME types for recognized image extensions. */
+const MIME_BY_EXT: Readonly<Record<string, string>> = {
+ ".png": "image/png",
+ ".jpg": "image/jpeg",
+ ".jpeg": "image/jpeg",
+ ".webp": "image/webp",
+ ".gif": "image/gif",
+ ".bmp": "image/bmp",
+};
+
+/** Reverse: MIME → extension. */
+const EXT_BY_MIME: Readonly<Record<string, string>> = {
+ "image/png": ".png",
+ "image/jpeg": ".jpg",
+ "image/webp": ".webp",
+ "image/gif": ".gif",
+ "image/bmp": ".bmp",
+};
+
+/**
+ * Read an image file from disk as a base64 data URL. Resolves relative paths
+ * against the cwd (the conversation's working directory). Throws on missing
+ * file / read error (the caller surfaces it). The shell edge — real `node:fs`.
+ */
+async function readFileAsDataUrl(path: string, cwd?: string): Promise<string> {
+ const abs = cwd !== undefined && !isAbsolute(path) ? pathResolve(cwd, path) : pathResolve(path);
+ const buf = await readFile(abs);
+ const ext = extname(abs).toLowerCase();
+ const mime = MIME_BY_EXT[ext] ?? "image/png";
+ return `data:${mime};base64,${buf.toString("base64")}`;
+}
+
+/**
+ * Save a data URL image to a tmp file and return a compact HTTP path.
+ * The compact URL (`/images/<conversationId>/<uuid>.<ext>`) is what gets
+ * persisted in the conversation store — a tiny string, not megabytes of base64.
+ */
+async function saveImageToTmp(
+ conversationId: string,
+ dataUrl: string,
+ mimeType?: string,
+): Promise<string> {
+ const mime = mimeType ?? "image/png";
+ const ext = EXT_BY_MIME[mime] ?? ".png";
+ const imageId = `${crypto.randomUUID()}${ext}`;
+ const dir = join(IMAGE_DIR, conversationId);
+ await mkdir(dir, { recursive: true });
+ const filePath = join(dir, imageId);
+ const base64 = dataUrl.split(",")[1] ?? "";
+ await writeFile(filePath, Buffer.from(base64, "base64"));
+ return `/images/${conversationId}/${imageId}`;
+}
+
+/**
+ * Resolve a compact URL (`/images/<convId>/<imageId>`) back to a data URL by
+ * reading the tmp file. Data URLs and HTTP URLs pass through unchanged.
+ */
+async function resolveImageUrl(url: string): Promise<string> {
+ if (url.startsWith("data:") || url.startsWith("http")) return url;
+ if (!url.startsWith("/images/")) return url;
+ const parts = url.split("/"); // ["", "images", convId, imageId]
+ const convId = parts[2];
+ const imageId = parts[3];
+ if (convId === undefined || imageId === undefined) return url;
+ const filePath = join(IMAGE_DIR, convId, imageId);
+ const buf = await readFile(filePath);
+ const ext = extname(imageId).toLowerCase();
+ const mime = MIME_BY_EXT[ext] ?? "image/png";
+ return `data:${mime};base64,${buf.toString("base64")}`;
+}
+
+/** Delete a single tmp image file (after compaction — best-effort). */
+async function deleteTmpImage(compactUrl: string): Promise<void> {
+ if (!compactUrl.startsWith("/images/")) return;
+ const parts = compactUrl.split("/");
+ const convId = parts[2];
+ const imageId = parts[3];
+ if (convId === undefined || imageId === undefined) return;
+ const filePath = join(IMAGE_DIR, convId, imageId);
+ try {
+ await unlink(filePath);
+ } catch {
+ // Best-effort — file may already be deleted.
+ }
+}
+
+/** Delete all tmp images for a conversation (on close — best-effort). */
+async function deleteConversationImages(conversationId: string): Promise<void> {
+ const dir = join(IMAGE_DIR, conversationId);
+ try {
+ await rm(dir, { recursive: true, force: true });
+ } catch {
+ // Best-effort.
+ }
+}
+
+export async function activate(host: HostAPI): Promise<void> {
+ const credentialStore = host.getService(credentialStoreHandle) as CredentialStore | undefined;
+ if (credentialStore === undefined) {
+ host.logger.warn(
+ "vision-handoff: credential-store service not available. The consult_vision tool and image handoff are disabled.",
+ );
+ return;
+ }
+
+ const resolveModel = (modelName: string) => {
+ const resolved = credentialStore.resolve(modelName);
+ if (resolved === undefined) return undefined;
+ const provider = host.getProviders().get(resolved.providerId);
+ if (provider === undefined) return undefined;
+ return { provider, model: resolved.model };
+ };
+
+ const service = createVisionHandoffService({
+ credentialStore,
+ resolveModel,
+ readFileAsDataUrl,
+ saveImageToTmp,
+ resolveImageUrl,
+ deleteTmpImage,
+ deleteConversationImages,
+ resolveOrchestrator: () => {
+ const loaded = host.getExtensions().some((m) => m.id === "session-orchestrator");
+ if (!loaded) return undefined;
+ try {
+ return host.getService(orchestratorLocalHandle);
+ } catch {
+ return undefined;
+ }
+ },
+ getImageTranscriptions: async (conversationId: string) => {
+ const store = host.getService(conversationStoreHandle);
+ return store.getImageTranscriptions(conversationId);
+ },
+ setImageTranscription: async (conversationId: string, url: string, text: string) => {
+ const store = host.getService(conversationStoreHandle);
+ await store.setImageTranscription(conversationId, url, text);
+ },
+ setConversationTitle: async (conversationId: string, title: string) => {
+ const store = host.getService(conversationStoreHandle);
+ await store.setConversationTitle(conversationId, title);
+ },
+ logger: host.logger.child({ extensionId: "vision-handoff" }),
+ });
+
+ host.provideService(visionHandoffHandle, service);
+ host.defineTool(createConsultVisionTool(service));
+ host.logger.info("vision-handoff: registered (consult_vision tool + handoff service)");
+}
+
+export const extension: Extension = { manifest, activate };
diff --git a/packages/vision-handoff/src/index.ts b/packages/vision-handoff/src/index.ts
new file mode 100644
index 0000000..2713346
--- /dev/null
+++ b/packages/vision-handoff/src/index.ts
@@ -0,0 +1,21 @@
+export { extension, manifest } from "./extension.js";
+export {
+ collectTextFromStream,
+ findVisionModelName,
+ formatConsultResult,
+ formatImagePlaceholder,
+ formatNoVisionPlaceholder,
+ isVisionCapable,
+} from "./pure.js";
+export type {
+ OrchestratorForVision,
+ ResolvedVisionModel,
+ VisionHandoffDeps,
+ VisionHandoffService,
+} from "./service.js";
+export {
+ createVisionHandoffService,
+ orchestratorLocalHandle,
+ visionHandoffHandle,
+} from "./service.js";
+export { createConsultVisionTool } from "./tool.js";
diff --git a/packages/vision-handoff/src/pure.test.ts b/packages/vision-handoff/src/pure.test.ts
new file mode 100644
index 0000000..21b1224
--- /dev/null
+++ b/packages/vision-handoff/src/pure.test.ts
@@ -0,0 +1,180 @@
+import type { ModelInfo, ProviderEvent } from "@dispatch/kernel";
+import { describe, expect, it } from "vitest";
+import {
+ collectTextFromStream,
+ findVisionModelName,
+ formatConsultationTitle,
+ formatConsultResult,
+ formatImagePlaceholder,
+ formatNoVisionPlaceholder,
+ isVisionCapable,
+} from "./pure.js";
+
+describe("isVisionCapable", () => {
+ it("returns true when ModelInfo.vision is true", () => {
+ expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: true })).toBe(
+ true,
+ );
+ });
+
+ it("returns false when ModelInfo.vision is false (overrides name heuristic)", () => {
+ expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: false })).toBe(
+ false,
+ );
+ });
+
+ it("falls back to name heuristic when vision is absent (umans kimi + qwen)", () => {
+ expect(isVisionCapable("umans/umans-kimi-k2.7", undefined)).toBe(true);
+ expect(isVisionCapable("umans/umans-qwen3.6-35b-a3b", undefined)).toBe(true);
+ });
+
+ it("falls back to name heuristic when vision is absent (non-vision)", () => {
+ expect(isVisionCapable("umans/umans-glm-5.2", undefined)).toBe(false);
+ expect(isVisionCapable("umans/umans-coder", { id: "umans-coder" })).toBe(false);
+ });
+
+ it("returns false for undefined model name", () => {
+ expect(isVisionCapable(undefined, undefined)).toBe(false);
+ });
+});
+
+describe("findVisionModelName", () => {
+ const getInfo = async (name: string): Promise<ModelInfo | undefined> => {
+ const map: Record<string, ModelInfo> = {
+ "umans/umans-kimi-k2.7": { id: "umans-kimi-k2.7", vision: true },
+ "umans/umans-qwen3.6-35b-a3b": { id: "umans-qwen3.6-35b-a3b", vision: true },
+ "umans/umans-glm-5.2": { id: "umans-glm-5.2" },
+ "umans/llama-vision": { id: "llama-vision", vision: true },
+ };
+ return map[name];
+ };
+
+ it("finds the first umans kimi model via name heuristic", async () => {
+ const name = await findVisionModelName(
+ ["umans/umans-glm-5.2", "umans/umans-kimi-k2.7", "umans/llama-vision"],
+ getInfo,
+ );
+ expect(name).toBe("umans/umans-kimi-k2.7");
+ });
+
+ it("finds a vision model via ModelInfo.vision when name heuristic misses", async () => {
+ const name = await findVisionModelName(["umans/umans-glm-5.2", "umans/llama-vision"], getInfo);
+ expect(name).toBe("umans/llama-vision");
+ });
+
+ it("skips the excluded model and finds the next vision model", async () => {
+ const name = await findVisionModelName(
+ ["umans/umans-kimi-k2.7", "umans/umans-qwen3.6-35b-a3b"],
+ getInfo,
+ "umans/umans-kimi-k2.7",
+ );
+ expect(name).toBe("umans/umans-qwen3.6-35b-a3b");
+ });
+
+ it("returns undefined when no vision model is available", async () => {
+ const name = await findVisionModelName(["umans/umans-glm-5.2"], getInfo);
+ expect(name).toBeUndefined();
+ });
+
+ it("returns undefined for empty catalog", async () => {
+ const name = await findVisionModelName([], getInfo);
+ expect(name).toBeUndefined();
+ });
+});
+
+describe("collectTextFromStream", () => {
+ async function* stream(events: ProviderEvent[]): AsyncIterable<ProviderEvent> {
+ for (const e of events) yield e;
+ }
+
+ it("collects text-delta events into a single string", async () => {
+ const events: ProviderEvent[] = [
+ { type: "text-delta", delta: "Hello " },
+ { type: "text-delta", delta: "world!" },
+ ];
+ const text = await collectTextFromStream(stream(events));
+ expect(text).toBe("Hello world!");
+ });
+
+ it("ignores non-text events", async () => {
+ const events: ProviderEvent[] = [
+ { type: "reasoning-delta", delta: "thinking..." },
+ { type: "text-delta", delta: "answer" },
+ { type: "usage", usage: { inputTokens: 5, outputTokens: 1 } },
+ { type: "finish", reason: "stop" },
+ ];
+ const text = await collectTextFromStream(stream(events));
+ expect(text).toBe("answer");
+ });
+
+ it("throws on an error event", async () => {
+ const events: ProviderEvent[] = [
+ { type: "text-delta", delta: "partial" },
+ { type: "error", message: "boom" },
+ ];
+ await expect(collectTextFromStream(stream(events))).rejects.toThrow("boom");
+ });
+
+ it("returns empty string for an empty stream", async () => {
+ const text = await collectTextFromStream(stream([]));
+ expect(text).toBe("");
+ });
+});
+
+describe("formatImagePlaceholder", () => {
+ it("includes the image ID and mentions consult_vision", () => {
+ const text = formatImagePlaceholder(1);
+ expect(text).toContain("Image 1");
+ expect(text).toContain("consult_vision");
+ expect(text).toContain("imageIds=[1]");
+ });
+
+ it("increments the ID for each image", () => {
+ expect(formatImagePlaceholder(2)).toContain("Image 2");
+ expect(formatImagePlaceholder(2)).toContain("imageIds=[2]");
+ });
+});
+
+describe("formatNoVisionPlaceholder", () => {
+ it("explains the limitation", () => {
+ const text = formatNoVisionPlaceholder();
+ expect(text).toContain("no vision-capable model");
+ });
+});
+
+describe("formatConsultResult", () => {
+ it("includes the conversation ID, the response, and the dispatch CLI hint", () => {
+ const result = formatConsultResult("abc-123", "The error is on line 12.");
+ expect(result).toContain("abc-123");
+ expect(result).toContain("The error is on line 12.");
+ expect(result).toContain("dispatch CLI");
+ });
+
+ it("trims the response", () => {
+ const result = formatConsultResult("c1", " spaced ");
+ expect(result).toContain("spaced");
+ expect(result).not.toContain("spaced ");
+ });
+});
+
+describe("formatConsultationTitle", () => {
+ it("prefixes the question with 'IMAGE - '", () => {
+ expect(formatConsultationTitle("What error is shown?")).toBe("IMAGE - What error is shown?");
+ });
+
+ it("truncates long questions to 80 chars with an ellipsis (matching the store's TITLE_MAX)", () => {
+ const long = "x".repeat(100);
+ const title = formatConsultationTitle(long);
+ expect(title).toBe(`IMAGE - ${"x".repeat(80)}…`);
+ expect(title.length).toBe("IMAGE - ".length + 80 + 1); // prefix + 80 + ellipsis
+ });
+
+ it("does not truncate questions at or under 80 chars", () => {
+ expect(formatConsultationTitle("x".repeat(80))).toBe(`IMAGE - ${"x".repeat(80)}`);
+ expect(formatConsultationTitle("x".repeat(79))).toBe(`IMAGE - ${"x".repeat(79)}`);
+ });
+
+ it("handles an empty question", () => {
+ expect(formatConsultationTitle("")).toBe("IMAGE - ");
+ });
+});
diff --git a/packages/vision-handoff/src/pure.ts b/packages/vision-handoff/src/pure.ts
new file mode 100644
index 0000000..af3476f
--- /dev/null
+++ b/packages/vision-handoff/src/pure.ts
@@ -0,0 +1,156 @@
+/**
+ * Pure decision helpers for the vision handoff.
+ *
+ * No I/O, no ambient state. The shell (the extension + the service) injects the
+ * effects (credential store lookups, orchestrator, provider streaming). This
+ * module owns only the policy: which model is vision-capable, how to format
+ * image placeholders for non-vision models, and how to format the
+ * consultation tool's result.
+ */
+
+import type { ModelInfo, ProviderEvent } from "@dispatch/kernel";
+import { isVisionModelId } from "@dispatch/openai-stream";
+
+/**
+ * Whether a model is vision-capable, given its catalog name and (optional)
+ * resolved `ModelInfo`. When `ModelInfo.vision` is present it is authoritative;
+ * otherwise fall back to the hardcoded name heuristic ({@link isVisionModelId}).
+ *
+ * The `modelName` is the `<credentialName>/<model>` catalog form; the heuristic
+ * inspects the model SEGMENT (after the first `/`) so `umans/kimi-k2.7` → the
+ * `kimi-k2.7` segment is checked. Pure.
+ */
+export function isVisionCapable(
+ modelName: string | undefined,
+ info: ModelInfo | undefined,
+): boolean {
+ // When ModelInfo explicitly reports vision (true OR false), it is authoritative
+ // — an explicit false overrides the name heuristic (a provider that KNOWS a
+ // model is non-vision wins over the name guess).
+ if (info?.vision !== undefined) return info.vision;
+ if (modelName === undefined) return false;
+ const slash = modelName.indexOf("/");
+ const modelId = slash >= 0 ? modelName.slice(slash + 1) : modelName;
+ return isVisionModelId(modelId);
+}
+
+/**
+ * Find the first vision-capable model name in a catalog, given a lookup that
+ * resolves a `<credentialName>/<model>` → `ModelInfo`. Returns `undefined` when
+ * no vision-capable model is available. Pure given the (async) lookup.
+ *
+ * @param catalog The full list of model names (`<credentialName>/<model>`).
+ * @param getInfo Async lookup of a model name → ModelInfo (from the credential store).
+ * @param exclude Optional model name to skip (e.g. the current non-vision model).
+ */
+export async function findVisionModelName(
+ catalog: readonly string[],
+ getInfo: (modelName: string) => Promise<ModelInfo | undefined>,
+ exclude?: string,
+): Promise<string | undefined> {
+ for (const name of catalog) {
+ if (exclude !== undefined && name === exclude) continue;
+ // Fast path: the name heuristic lets us short-circuit without an async
+ // lookup for known vision families (kimi).
+ const slash = name.indexOf("/");
+ const modelId = slash >= 0 ? name.slice(slash + 1) : name;
+ if (isVisionModelId(modelId)) return name;
+ const info = await getInfo(name);
+ if (info?.vision === true) return name;
+ }
+ return undefined;
+}
+
+/**
+ * Fold a provider's streamed events into a single text string. Pure given the
+ * async iterable — collects `text-delta` events, ignores everything else
+ * (reasoning, usage, tool-calls). If the stream yields an error event, it is
+ * surfaced as a thrown Error so the caller can decide how to degrade.
+ */
+export async function collectTextFromStream(stream: AsyncIterable<ProviderEvent>): Promise<string> {
+ let text = "";
+ for await (const event of stream) {
+ if (event.type === "text-delta") {
+ text += event.delta;
+ } else if (event.type === "error") {
+ throw new Error(event.message);
+ }
+ }
+ return text;
+}
+
+/**
+ * Format the placeholder text that replaces an `image` chunk when a non-vision
+ * model is active. The placeholder tells the model an image is attached and it
+ * should call `consult_vision` to analyze it — the model drives the analysis
+ * (asking a specific question) rather than receiving a pre-emptive generic dump.
+ *
+ * @param imageId The 1-based ID assigned to this image (used by the tool to
+ * look up the registered image data).
+ * Pure.
+ */
+export function formatImagePlaceholder(imageId: number): string {
+ return (
+ `[Image ${imageId} attached — you cannot view images. Call the ` +
+ `consult_vision tool with imageIds=[${imageId}] and a specific question ` +
+ `to analyze it via a vision-capable model.]`
+ );
+}
+
+/**
+ * Placeholder text used when NO vision-capable model is available (the
+ * degraded path — the tool cannot function). Pure.
+ */
+export function formatNoVisionPlaceholder(): string {
+ return (
+ "[Image attached — no vision-capable model is available to analyze it. " +
+ "Install or configure a vision-capable model (e.g. kimi) to enable image analysis.]"
+ );
+}
+
+/**
+ * Maximum length of the consultation title body (matching the conversation
+ * store's `TITLE_MAX`). The question is truncated to this before the
+ * `"IMAGE - "` prefix is applied so the consultation tab's title stays in line
+ * with the store's own title-derivation limit.
+ */
+const CONSULTATION_TITLE_MAX = 80;
+
+/**
+ * Format the title for a vision consultation conversation tab. The title is
+ * `"IMAGE - "` prefixed to the (truncated) question so the tab is visually
+ * distinguishable from normal conversation tabs. The question is truncated to
+ * match the conversation store's title-derivation limit (`TITLE_MAX = 80`).
+ *
+ * Pure.
+ *
+ * @param question The question the model asked the vision model.
+ */
+export function formatConsultationTitle(question: string): string {
+ const body =
+ question.length > CONSULTATION_TITLE_MAX
+ ? `${question.slice(0, CONSULTATION_TITLE_MAX)}…`
+ : question;
+ return `IMAGE - ${body}`;
+}
+
+/**
+ * Format the `consult_vision` tool's result string. Returns the conversation ID
+ * (so the model / user can continue the vision consultation), the vision model's
+ * response, and a note that follow-up questions use the dispatch CLI (the model
+ * can load the `dispatch-cli` skill for the exact commands).
+ *
+ * Pure.
+ *
+ * @param conversationId The new vision consultation conversation ID.
+ * @param response The vision model's answer to the model's question.
+ */
+export function formatConsultResult(conversationId: string, response: string): string {
+ const trimmed = response.trim();
+ return (
+ `Vision consultation opened in conversation ${conversationId}.\n\n` +
+ `Response: ${trimmed}\n\n` +
+ `To ask follow-up questions about this image, use the dispatch CLI ` +
+ `(conversation: ${conversationId}).`
+ );
+}
diff --git a/packages/vision-handoff/src/service.test.ts b/packages/vision-handoff/src/service.test.ts
new file mode 100644
index 0000000..8c4117e
--- /dev/null
+++ b/packages/vision-handoff/src/service.test.ts
@@ -0,0 +1,375 @@
+import type {
+ AgentEvent,
+ ChatMessage,
+ ModelInfo,
+ ProviderContract,
+ ProviderEvent,
+ ToolContract,
+} from "@dispatch/kernel";
+import { describe, expect, it, vi } from "vitest";
+import { createVisionHandoffService, type VisionHandoffDeps } from "./service.js";
+
+// ── Test doubles (outermost-edge fakes — NOT @dispatch/* mocks) ──────────────
+
+function makeVisionProvider(
+ describe: (imageUrl: string) => string,
+ id = "umans",
+): ProviderContract {
+ return {
+ id,
+ stream: vi.fn(
+ (
+ messages: readonly ChatMessage[],
+ _tools: readonly ToolContract[],
+ ): AsyncIterable<ProviderEvent> => {
+ const img = messages.flatMap((m) => m.chunks).find((c) => c.type === "image");
+ const url = img && img.type === "image" ? img.url : "";
+ const text = describe(url);
+ async function* gen(): AsyncIterable<ProviderEvent> {
+ yield { type: "text-delta", delta: text };
+ yield { type: "finish", reason: "stop" };
+ }
+ return gen();
+ },
+ ),
+ };
+}
+
+function makeDeps(overrides: Partial<VisionHandoffDeps> = {}): VisionHandoffDeps {
+ const visionProvider = makeVisionProvider((url) => `DESCRIPTION of ${url}`);
+ const catalog = ["umans/umans-kimi-k2.7", "umans/umans-glm-5.2"];
+ const infoMap: Record<string, ModelInfo> = {
+ "umans/umans-kimi-k2.7": { id: "umans-kimi-k2.7", vision: true },
+ "umans/umans-glm-5.2": { id: "umans-glm-5.2" },
+ };
+ return {
+ credentialStore: {
+ listCatalog: vi.fn(async () => catalog),
+ getModelInfo: vi.fn(async (name: string) => infoMap[name]),
+ resolve: vi.fn((name: string) => {
+ if (name === "umans/umans-kimi-k2.7")
+ return { providerId: "umans", model: "umans-kimi-k2.7" };
+ if (name === "umans/umans-glm-5.2") return { providerId: "umans", model: "umans-glm-5.2" };
+ return undefined;
+ }),
+ },
+ resolveModel: vi.fn((name: string) =>
+ name === "umans/umans-kimi-k2.7" || name === "umans/umans-glm-5.2"
+ ? { provider: visionProvider, model: name.split("/")[1] }
+ : undefined,
+ ),
+ readFileAsDataUrl: vi.fn(async (path: string) => `data:image/png;base64,FILE(${path})`),
+ setConversationTitle: vi.fn(async (_conversationId: string, _title: string) => {}),
+ ...overrides,
+ };
+}
+
+describe("VisionHandoffService.isVisionCapable", () => {
+ it("returns true for kimi (via ModelInfo)", async () => {
+ const svc = createVisionHandoffService(makeDeps());
+ expect(await svc.isVisionCapable("umans/umans-kimi-k2.7")).toBe(true);
+ });
+
+ it("returns false for glm-5.2", async () => {
+ const svc = createVisionHandoffService(makeDeps());
+ expect(await svc.isVisionCapable("umans/umans-glm-5.2")).toBe(false);
+ });
+
+ it("returns false for undefined model name", async () => {
+ const svc = createVisionHandoffService(makeDeps());
+ expect(await svc.isVisionCapable(undefined)).toBe(false);
+ });
+});
+
+describe("VisionHandoffService.resolveVisionModel", () => {
+ it("resolves the kimi model from the catalog", async () => {
+ const svc = createVisionHandoffService(makeDeps());
+ const vision = await svc.resolveVisionModel();
+ expect(vision?.modelName).toBe("umans/umans-kimi-k2.7");
+ expect(vision?.model).toBe("umans-kimi-k2.7");
+ });
+
+ it("excludes the given model", async () => {
+ const svc = createVisionHandoffService(makeDeps());
+ const vision = await svc.resolveVisionModel("umans/umans-kimi-k2.7");
+ expect(vision).toBeUndefined();
+ });
+});
+
+describe("VisionHandoffService.prepareForProvider", () => {
+ it("passes messages through unchanged when the model is vision-capable", async () => {
+ const deps = makeDeps();
+ const svc = createVisionHandoffService(deps);
+ const messages: ChatMessage[] = [
+ {
+ role: "user",
+ chunks: [
+ { type: "text", text: "What's this?" },
+ { type: "image", url: "data:image/png;base64,abc" },
+ ],
+ },
+ ];
+ const result = await svc.prepareForProvider(messages, "umans/umans-kimi-k2.7");
+ expect(result).toBe(messages); // same reference — no copy, no change
+ });
+
+ it("passes messages through unchanged when there are no images", async () => {
+ const deps = makeDeps();
+ const svc = createVisionHandoffService(deps);
+ const messages: ChatMessage[] = [{ role: "user", chunks: [{ type: "text", text: "hi" }] }];
+ const result = await svc.prepareForProvider(messages, "umans/umans-glm-5.2");
+ expect(result).toBe(messages);
+ });
+
+ it("replaces image chunks with numbered placeholders for a non-vision model", async () => {
+ const deps = makeDeps();
+ const svc = createVisionHandoffService(deps);
+ const messages: ChatMessage[] = [
+ {
+ role: "user",
+ chunks: [
+ { type: "text", text: "Describe this" },
+ { type: "image", url: "data:image/png;base64,img1" },
+ ],
+ },
+ ];
+ const result = await svc.prepareForProvider(messages, "umans/umans-glm-5.2", {
+ conversationId: "conv-1",
+ });
+ expect(result).toHaveLength(1);
+ const chunks = result[0]?.chunks;
+ expect(chunks).toHaveLength(2);
+ // Text chunk unchanged.
+ expect(chunks?.[0]).toEqual({ type: "text", text: "Describe this" });
+ // Image chunk → placeholder text.
+ expect(chunks?.[1]?.type).toBe("text");
+ const placeholder = (chunks?.[1] as { text: string }).text;
+ expect(placeholder).toContain("Image 1");
+ expect(placeholder).toContain("consult_vision");
+ });
+
+ it("assigns sequential image IDs across multiple messages", async () => {
+ const deps = makeDeps();
+ const svc = createVisionHandoffService(deps);
+ const messages: ChatMessage[] = [
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,a" }] },
+ { role: "assistant", chunks: [{ type: "text", text: "ok" }] },
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,b" }] },
+ ];
+ const result = await svc.prepareForProvider(messages, "umans/umans-glm-5.2", {
+ conversationId: "conv-1",
+ });
+ // First image → Image 1, second → Image 2.
+ expect((result[0]?.chunks[0] as { text: string }).text).toContain("Image 1");
+ // Assistant message unchanged.
+ expect(result[1]?.chunks[0]?.type).toBe("text");
+ expect((result[2]?.chunks[0] as { text: string }).text).toContain("Image 2");
+ });
+
+ it("registers images so getRegisteredImage can look them up", async () => {
+ const deps = makeDeps();
+ const svc = createVisionHandoffService(deps);
+ const messages: ChatMessage[] = [
+ {
+ role: "user",
+ chunks: [{ type: "image", url: "data:image/png;base64,registered" }],
+ },
+ ];
+ await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { conversationId: "conv-42" });
+ const img = svc.getRegisteredImage("conv-42", 1);
+ expect(img?.url).toBe("data:image/png;base64,registered");
+ });
+
+ it("uses no-vision placeholder when no vision model is available", async () => {
+ const deps = makeDeps();
+ (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]);
+ const svc = createVisionHandoffService(deps);
+ const messages: ChatMessage[] = [
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,abc" }] },
+ ];
+ const result = await svc.prepareForProvider(messages, "umans/umans-glm-5.2", {
+ conversationId: "conv-1",
+ });
+ const text = (result[0]?.chunks[0] as { text: string }).text;
+ expect(text).toContain("no vision-capable model");
+ expect(text).not.toContain("consult_vision");
+ });
+});
+
+describe("VisionHandoffService.consultVision", () => {
+ function makeOrchestratorDouble(response: string): {
+ orchestrator: NonNullable<
+ VisionHandoffDeps["resolveOrchestrator"] extends () => infer T ? T : never
+ >;
+ handleMessage: ReturnType<typeof vi.fn>;
+ } {
+ const handleMessage = vi.fn(
+ async (input: {
+ conversationId: string;
+ text: string;
+ onEvent: (event: AgentEvent) => void;
+ }): Promise<void> => {
+ input.onEvent({
+ type: "text-delta",
+ conversationId: input.conversationId,
+ turnId: "t1",
+ delta: response,
+ });
+ input.onEvent({
+ type: "done",
+ conversationId: input.conversationId,
+ turnId: "t1",
+ reason: "stop",
+ });
+ },
+ );
+ return { orchestrator: { handleMessage }, handleMessage };
+ }
+
+ it("opens a new consultation with a pasted image and returns convId + response", async () => {
+ const deps = makeDeps();
+ const { orchestrator, handleMessage } = makeOrchestratorDouble("The error is on line 12.");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+
+ // Register an image first (as prepareForProvider would).
+ const messages: ChatMessage[] = [
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,img1" }] },
+ ];
+ await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { conversationId: "conv-1" });
+
+ const result = await svc.consultVision("What error is shown?", {
+ conversationId: "conv-1",
+ imageIds: [1],
+ });
+
+ expect("error" in result).toBe(false);
+ if (!("error" in result)) {
+ expect(result.conversationId).toBeTruthy();
+ expect(result.response).toContain("line 12");
+ expect(result.response).toContain(result.conversationId);
+ expect(result.response).toContain("dispatch CLI");
+ }
+ // The orchestrator was called with the vision model + the image.
+ expect(handleMessage).toHaveBeenCalledOnce();
+ const call = handleMessage.mock.calls[0]?.[0];
+ expect(call.modelName).toBe("umans/umans-kimi-k2.7");
+ expect(call.images).toHaveLength(1);
+ expect(call.images?.[0]?.url).toBe("data:image/png;base64,img1");
+ });
+
+ it("labels the consultation tab with an 'IMAGE - ' prefixed title", async () => {
+ const deps = makeDeps();
+ const { orchestrator } = makeOrchestratorDouble("The error is on line 12.");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+
+ // Register an image first (as prepareForProvider would).
+ const messages: ChatMessage[] = [
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,img1" }] },
+ ];
+ await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { conversationId: "conv-1" });
+
+ const result = await svc.consultVision("What error is shown?", {
+ conversationId: "conv-1",
+ imageIds: [1],
+ });
+
+ expect("error" in result).toBe(false);
+ // The title was set with the IMAGE - prefix + the question.
+ expect(deps.setConversationTitle).toHaveBeenCalledOnce();
+ const [titleConvId, title] = (deps.setConversationTitle as ReturnType<typeof vi.fn>).mock
+ .calls[0];
+ expect(titleConvId).toBe((result as { conversationId: string }).conversationId);
+ expect(title).toBe("IMAGE - What error is shown?");
+ });
+
+ it("does not call setConversationTitle when it is not provided", async () => {
+ const deps = makeDeps({ setConversationTitle: undefined });
+ const { orchestrator } = makeOrchestratorDouble("response");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+
+ const messages: ChatMessage[] = [
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,img1" }] },
+ ];
+ await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { conversationId: "conv-1" });
+
+ // Should NOT throw — setConversationTitle is optional.
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ imageIds: [1],
+ });
+ expect("error" in result).toBe(false);
+ });
+
+ it("opens a consultation with a file path image", async () => {
+ const deps = makeDeps();
+ const { orchestrator } = makeOrchestratorDouble("It's a diagram.");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+
+ const result = await svc.consultVision("What is this diagram?", {
+ conversationId: "conv-1",
+ path: "diagram.png",
+ cwd: "/work",
+ });
+
+ expect("error" in result).toBe(false);
+ expect(deps.readFileAsDataUrl).toHaveBeenCalledWith("diagram.png", "/work");
+ });
+
+ it("returns an error when imageId is not registered", async () => {
+ const deps = makeDeps();
+ const { orchestrator } = makeOrchestratorDouble("response");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ imageIds: [99], // not registered
+ });
+ expect("error" in result).toBe(true);
+ if ("error" in result) {
+ expect(result.error).toContain("Image 99");
+ }
+ });
+
+ it("returns an error when no orchestrator is available", async () => {
+ const deps = makeDeps();
+ // No resolveOrchestrator provided.
+ const svc = createVisionHandoffService(deps);
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ imageIds: [1],
+ });
+ expect("error" in result).toBe(true);
+ });
+
+ it("returns an error when no vision model is available", async () => {
+ const deps = makeDeps();
+ (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]);
+ const { orchestrator } = makeOrchestratorDouble("response");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ imageIds: [1],
+ });
+ expect("error" in result).toBe(true);
+ if ("error" in result) {
+ expect(result.error).toContain("No vision-capable model");
+ }
+ });
+
+ it("returns an error when no image source is provided", async () => {
+ const deps = makeDeps();
+ const { orchestrator } = makeOrchestratorDouble("response");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ });
+ expect("error" in result).toBe(true);
+ });
+});
diff --git a/packages/vision-handoff/src/service.ts b/packages/vision-handoff/src/service.ts
new file mode 100644
index 0000000..397d81a
--- /dev/null
+++ b/packages/vision-handoff/src/service.ts
@@ -0,0 +1,689 @@
+/**
+ * Vision handoff service — the imperative shell that performs the universal,
+ * provider-agnostic vision handoff.
+ *
+ * Two capabilities:
+ * 1. **prepareForProvider** (`prepareForProvider`): when a user message carries
+ * images but the active model cannot see them, this replaces each image chunk
+ * with a numbered placeholder (telling the model to call `consult_vision`)
+ * and registers the image data in a per-conversation registry for tool
+ * access. Vision-capable models pass through unchanged (images flow natively).
+ * 2. **consult_vision tool** (`consultVision`): opens a NEW conversation tab with
+ * a vision-capable model (resolved from the catalog — any provider), attaches
+ * the image(s) + the model's specific question, waits for the response, and
+ * returns the conversation ID + the vision model's answer. The model (e.g.
+ * GLM 5.2) directs the analysis — asking exactly what it needs — instead of
+ * receiving a pre-emptive generic dump. Follow-up questions go through the
+ * dispatch CLI (the conversation ID is the bridge), not another tool call.
+ *
+ * Effects (credential store, orchestrator, filesystem) are injected. The pure
+ * decisions live in `pure.ts`. This shell wires them.
+ */
+
+import type { CredentialStore } from "@dispatch/credential-store";
+import type {
+ AgentEvent,
+ ChatMessage,
+ Chunk,
+ ImageInput,
+ Logger,
+ ModelInfo,
+ ProviderContract,
+} from "@dispatch/kernel";
+import { defineService, type ServiceHandle } from "@dispatch/kernel";
+import {
+ collectTextFromStream,
+ findVisionModelName,
+ formatConsultationTitle,
+ formatConsultResult,
+ formatImagePlaceholder,
+ formatNoVisionPlaceholder,
+ isVisionCapable,
+} from "./pure.js";
+
+/**
+ * Minimal orchestrator interface the service needs to start vision consultation
+ * turns. Defined locally (not imported from session-orchestrator) to avoid a
+ * compile-time dependency — resolved lazily at runtime via a local handle keyed
+ * to the same service ID.
+ */
+export interface OrchestratorForVision {
+ readonly handleMessage: (input: {
+ readonly conversationId: string;
+ readonly text: string;
+ readonly onEvent: (event: AgentEvent) => void;
+ readonly modelName?: string;
+ readonly cwd?: string;
+ readonly images?: readonly ImageInput[];
+ readonly systemPrompt?: string;
+ }) => Promise<void>;
+}
+
+/** Local handle for the session-orchestrator service (same ID, no import dep). */
+export const orchestratorLocalHandle: ServiceHandle<OrchestratorForVision> =
+ defineService<OrchestratorForVision>("session-orchestrator/orchestrator");
+
+/**
+ * Resolved vision model — a provider + its model id, ready to stream from.
+ */
+export interface ResolvedVisionModel {
+ readonly provider: ProviderContract;
+ readonly model: string;
+ readonly modelName: string;
+}
+
+/** A registered image (looked up by the consult_vision tool via imageId). */
+interface RegisteredImage {
+ readonly url: string;
+ readonly mimeType?: string;
+}
+
+/**
+ * Dependencies the service needs — all injected (no ambient state).
+ */
+export interface VisionHandoffDeps {
+ readonly credentialStore: CredentialStore;
+ /** Resolve a `<credentialName>/<model>` → its provider + model id. */
+ readonly resolveModel: (
+ modelName: string,
+ ) => { provider: ProviderContract; model: string } | undefined;
+ /**
+ * Read a file from disk as a base64 data URL. Injected so the shell controls
+ * the filesystem edge. Returns the data URL, or throws on error.
+ */
+ readonly readFileAsDataUrl: (path: string, cwd?: string) => Promise<string>;
+ /**
+ * Lazily resolve the session-orchestrator (for starting vision consultation
+ * turns). Returns `undefined` when not available — `consult_vision` degrades
+ * with an error. Lazy so activation order doesn't matter.
+ */
+ readonly resolveOrchestrator?: () => OrchestratorForVision | undefined;
+ /**
+ * Get the per-conversation cached image transcriptions (imageUrl → text).
+ * Used to avoid re-transcribing old images that were compacted to text on a
+ * previous turn. Optional — when absent, compaction still works but
+ * re-transcribes every turn (no caching).
+ */
+ readonly getImageTranscriptions?: (
+ conversationId: string,
+ ) => Promise<ReadonlyMap<string, string>>;
+ /**
+ * Upsert a single image transcription into the per-conversation cache.
+ * Optional — paired with getImageTranscriptions.
+ */
+ readonly setImageTranscription?: (
+ conversationId: string,
+ imageUrl: string,
+ transcription: string,
+ ) => Promise<void>;
+ /**
+ * Save an image data URL to a tmp file and return a compact URL
+ * (`/images/<conversationId>/<imageId>.<ext>`) that can be persisted in the
+ * conversation store instead of the full data URL (which would be megabytes).
+ * The frontend serves the image via `GET /images/...`; the provider resolves
+ * it back to a data URL via {@link resolveImageUrl} at runtime. When `undefined`,
+ * data URLs pass through unchanged (images persist in SQLite — the large-DB
+ * path, for environments without tmp file support).
+ */
+ readonly saveImageToTmp?: (
+ conversationId: string,
+ dataUrl: string,
+ mimeType?: string,
+ ) => Promise<string>;
+ /**
+ * Resolve a compact URL (`/images/...`) back to a data URL by reading the tmp
+ * file. Data URLs and HTTP URLs pass through unchanged. Paired with
+ * {@link saveImageToTmp}.
+ */
+ readonly resolveImageUrl?: (url: string) => Promise<string>;
+ /**
+ * Delete a tmp image file (after it has been compacted to text — the
+ * transcription is cached, the raw image is no longer needed). Best-effort:
+ * errors are logged, not thrown.
+ */
+ readonly deleteTmpImage?: (compactUrl: string) => Promise<void>;
+ /**
+ * Delete all tmp images for a conversation (on conversation close).
+ * Best-effort.
+ */
+ readonly deleteConversationImages?: (conversationId: string) => Promise<void>;
+ /**
+ * Set the human-readable title of a conversation. Used to label vision
+ * consultation tabs with an `"IMAGE - "` prefix so they're visually
+ * distinguishable from normal conversation tabs. Backed by the conversation
+ * store's `setConversationTitle`. Optional — when absent, consultation tabs
+ * keep their default (question-derived) title.
+ */
+ readonly setConversationTitle?: (conversationId: string, title: string) => Promise<void>;
+ /** Generate a new conversation ID for a consultation. Defaults to crypto.randomUUID. */
+ readonly generateId?: () => string;
+ readonly logger?: Logger;
+}
+
+export interface VisionHandoffService {
+ /**
+ * Whether a given model (by catalog name) is vision-capable. Uses the
+ * credential store's ModelInfo + the name heuristic.
+ */
+ readonly isVisionCapable: (modelName: string | undefined) => Promise<boolean>;
+
+ /**
+ * Store images to tmp files and return compact URLs. Each input image's data
+ * URL is saved to `/tmp/dispatch/images/<conversationId>/<uuid>.<ext>` and
+ * replaced with a compact HTTP path (`/images/<conversationId>/<uuid>.<ext>`)
+ * so the persisted conversation store holds a tiny string, not megabytes of
+ * base64. When `saveImageToTmp` is not configured, data URLs pass through
+ * unchanged (backward compatible).
+ */
+ readonly storeImages: (
+ conversationId: string,
+ images: readonly ImageInput[],
+ ) => Promise<readonly ImageInput[]>;
+
+ /**
+ * Delete all tmp images for a conversation (on close). Best-effort.
+ */
+ readonly purgeConversationImages: (conversationId: string) => Promise<void>;
+
+ /**
+ * Resolve a vision-capable model from the catalog (any provider). Returns
+ * `undefined` when none is available.
+ */
+ readonly resolveVisionModel: (excludeName?: string) => Promise<ResolvedVisionModel | undefined>;
+
+ /**
+ * Transform a message list for the provider: if the active model is
+ * vision-capable, return messages unchanged (images pass through natively).
+ * If NOT vision-capable, replace every `image` chunk with a numbered
+ * placeholder (telling the model to call `consult_vision`) and register the
+ * image data in the per-conversation registry for tool access. The PERSISTED
+ * history is NOT modified — only what the provider sees. Never throws.
+ */
+ readonly prepareForProvider: (
+ messages: readonly ChatMessage[],
+ currentModelName: string | undefined,
+ opts?: {
+ readonly conversationId?: string;
+ readonly imageLimit?: number;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
+ ) => Promise<readonly ChatMessage[]>;
+
+ /**
+ * Look up a registered image by conversation ID + image ID. Returns
+ * `undefined` when the image isn't registered (e.g. after a server restart).
+ */
+ readonly getRegisteredImage: (
+ conversationId: string,
+ imageId: number,
+ ) => RegisteredImage | undefined;
+
+ /**
+ * Open a NEW vision consultation conversation: attach image(s) + the model's
+ * question to a vision-capable model, wait for the response, and return the
+ * conversation ID + the vision model's answer. The model drives the analysis
+ * — it asks exactly what it needs. Follow-ups go through the dispatch CLI.
+ *
+ * @returns The conversation ID + the vision model's response text, or an
+ * error string (never throws — the tool surfaces it).
+ */
+ readonly consultVision: (
+ question: string,
+ opts: {
+ readonly conversationId: string;
+ readonly imageIds?: readonly number[];
+ readonly path?: string;
+ readonly cwd?: string;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
+ ) => Promise<
+ { readonly conversationId: string; readonly response: string } | { readonly error: string }
+ >;
+}
+
+export const visionHandoffHandle: ServiceHandle<VisionHandoffService> =
+ defineService<VisionHandoffService>("vision-handoff/service");
+
+/** Whether a message list contains any image chunks. Pure. */
+function hasImageChunks(messages: readonly ChatMessage[]): boolean {
+ return messages.some((m) => m.chunks.some((c) => c.type === "image"));
+}
+
+export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHandoffService {
+ const log = deps.logger;
+ const generateId = deps.generateId ?? (() => crypto.randomUUID());
+
+ // Per-conversation image registry: conversationId → (imageId → image data).
+ // Populated by prepareForProvider; consulted by the consult_vision tool.
+ // In-memory only (cleared on restart — the user re-pastes if needed).
+ const imageRegistry = new Map<string, Map<number, RegisteredImage>>();
+
+ async function getInfo(modelName: string): Promise<ModelInfo | undefined> {
+ return deps.credentialStore.getModelInfo(modelName);
+ }
+
+ async function resolveVisionModel(
+ excludeName?: string,
+ ): Promise<ResolvedVisionModel | undefined> {
+ const catalog = await deps.credentialStore.listCatalog();
+ const name = await findVisionModelName(catalog, getInfo, excludeName);
+ if (name === undefined) return undefined;
+ const resolved = deps.resolveModel(name);
+ if (resolved === undefined) return undefined;
+ return { provider: resolved.provider, model: resolved.model, modelName: name };
+ }
+
+ /**
+ * Compact images for a vision-capable model: when the conversation has more
+ * image chunks than the limit, the oldest images are transcribed to text
+ * (one-time, cached in the conversation store) and stripped from the
+ * provider messages. Recent images (within the limit) stay native.
+ *
+ * The persisted history is NOT modified — only the provider's view.
+ * Transcriptions are cached so they're reused on subsequent turns (no
+ * re-transcription). When no caching deps are available, it still works but
+ * re-transcribes every turn.
+ */
+ async function compactImagesForVisionModel(
+ messages: readonly ChatMessage[],
+ opts:
+ | {
+ readonly conversationId?: string;
+ readonly imageLimit?: number;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ }
+ | undefined,
+ currentModelName: string | undefined,
+ ): Promise<readonly ChatMessage[]> {
+ void currentModelName; // reserved for future model-specific compaction logic
+ const limit = opts?.imageLimit;
+ // No limit or limit <= 0 → pass all images through (compaction disabled).
+ if (limit === undefined || limit <= 0) return messages;
+
+ // Collect all image chunks in order (oldest first, across all messages).
+ const imageEntries: { msgIdx: number; chunkIdx: number; url: string }[] = [];
+ for (const [mi, msg] of messages.entries()) {
+ for (const [ci, chunk] of msg.chunks.entries()) {
+ if (chunk.type === "image") {
+ imageEntries.push({ msgIdx: mi, chunkIdx: ci, url: chunk.url });
+ }
+ }
+ }
+
+ // If within the limit, pass everything through natively.
+ if (imageEntries.length <= limit) return messages;
+
+ // The oldest (imageEntries.length - limit) images need transcription.
+ const toTranscribeCount = imageEntries.length - limit;
+ const toTranscribe = imageEntries.slice(0, toTranscribeCount);
+
+ // Load cached transcriptions.
+ const convId = opts?.conversationId;
+ const cache =
+ convId !== undefined && deps.getImageTranscriptions !== undefined
+ ? await deps.getImageTranscriptions(convId)
+ : new Map<string, string>();
+
+ // Transcribe any that aren't cached yet (via the vision model).
+ const transcriptions = new Map<string, string>(cache);
+ const vision = await resolveVisionModel();
+ for (const entry of toTranscribe) {
+ if (transcriptions.has(entry.url)) continue;
+ if (vision === undefined) {
+ // No vision model available for transcription — use a placeholder.
+ transcriptions.set(
+ entry.url,
+ "[Image was compacted — no vision model available to transcribe it.]",
+ );
+ continue;
+ }
+ try {
+ const prompt =
+ "Describe this image in detail. Include visible text (transcribe verbatim), " +
+ "key objects, layout, and notable details. This description will replace " +
+ "the image in a conversation history, so be thorough.";
+ const userMessage: ChatMessage = {
+ role: "user",
+ chunks: [
+ { type: "text", text: prompt },
+ { type: "image", url: entry.url },
+ ],
+ };
+ const stream = vision.provider.stream([userMessage], [], {
+ model: vision.model,
+ systemPrompt:
+ "You are a vision assistant. Describe images faithfully and thoroughly. " +
+ "Do not use any tools — just use your vision to see the image and describe it directly.",
+ });
+ const description = (await collectTextFromStream(stream)).trim();
+ const text =
+ description.length > 0 ? description : "[Image transcription produced no output.]";
+ transcriptions.set(entry.url, text);
+ // Cache it in the conversation store (if available).
+ if (convId !== undefined && deps.setImageTranscription !== undefined) {
+ await deps.setImageTranscription(convId, entry.url, text);
+ }
+ // The image has been transcribed to text — delete the tmp file
+ // (the transcription is cached, the raw image is no longer needed).
+ if (deps.deleteTmpImage !== undefined) {
+ try {
+ await deps.deleteTmpImage(entry.url);
+ } catch {
+ // Best-effort — don't let cleanup failure break the turn.
+ }
+ }
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ log?.warn("vision-handoff: image compaction transcription failed", { error: msg });
+ transcriptions.set(entry.url, `[Image transcription failed: ${msg}]`);
+ }
+ }
+
+ // Build the provider messages: replace transcribed images with text,
+ // keep recent images (within the limit) native.
+ const transcribedUrls = new Set(toTranscribe.map((e) => e.url));
+ const result: ChatMessage[] = [];
+ for (const msg of messages) {
+ if (!msg.chunks.some((c) => c.type === "image")) {
+ result.push(msg);
+ continue;
+ }
+ const newChunks: Chunk[] = [];
+ for (const chunk of msg.chunks) {
+ if (chunk.type === "image" && transcribedUrls.has(chunk.url)) {
+ const transcription = transcriptions.get(chunk.url);
+ if (transcription !== undefined) {
+ newChunks.push({ type: "text", text: `[Compacted image]: ${transcription}` });
+ } else {
+ newChunks.push(chunk); // fallback: keep the image
+ }
+ } else {
+ newChunks.push(chunk);
+ }
+ }
+ result.push({ role: msg.role, chunks: newChunks });
+ }
+ return result;
+ }
+
+ async function resolveImageUrlsInMessages(
+ messages: readonly ChatMessage[],
+ ): Promise<readonly ChatMessage[]> {
+ if (deps.resolveImageUrl === undefined) return messages;
+ let hasCompact = false;
+ for (const msg of messages) {
+ if (msg.chunks.some((c) => c.type === "image")) {
+ hasCompact = true;
+ break;
+ }
+ }
+ if (!hasCompact) return messages;
+ const result: ChatMessage[] = [];
+ for (const msg of messages) {
+ if (!msg.chunks.some((c) => c.type === "image")) {
+ result.push(msg);
+ continue;
+ }
+ const newChunks: Chunk[] = [];
+ for (const chunk of msg.chunks) {
+ if (chunk.type === "image") {
+ const dataUrl = await deps.resolveImageUrl!(chunk.url);
+ newChunks.push({
+ type: "image",
+ url: dataUrl,
+ ...(chunk.mimeType !== undefined ? { mimeType: chunk.mimeType } : {}),
+ });
+ } else {
+ newChunks.push(chunk);
+ }
+ }
+ result.push({ role: msg.role, chunks: newChunks });
+ }
+ return result;
+ }
+
+ const service: VisionHandoffService = {
+ async isVisionCapable(modelName: string | undefined): Promise<boolean> {
+ if (modelName === undefined) return false;
+ const info = await getInfo(modelName);
+ return isVisionCapable(modelName, info);
+ },
+
+ async storeImages(
+ conversationId: string,
+ images: readonly ImageInput[],
+ ): Promise<readonly ImageInput[]> {
+ if (deps.saveImageToTmp === undefined) return images;
+ const result: ImageInput[] = [];
+ for (const img of images) {
+ if (img.url.startsWith("data:")) {
+ const compactUrl = await deps.saveImageToTmp(conversationId, img.url, img.mimeType);
+ result.push({
+ url: compactUrl,
+ ...(img.mimeType !== undefined ? { mimeType: img.mimeType } : {}),
+ });
+ } else {
+ result.push(img);
+ }
+ }
+ return result;
+ },
+
+ async purgeConversationImages(conversationId: string): Promise<void> {
+ if (deps.deleteConversationImages === undefined) return;
+ try {
+ await deps.deleteConversationImages(conversationId);
+ } catch (err) {
+ log?.warn("vision-handoff: failed to purge conversation images", {
+ conversationId,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ }
+ },
+
+ resolveVisionModel,
+
+ async prepareForProvider(
+ messages: readonly ChatMessage[],
+ currentModelName: string | undefined,
+ opts?: {
+ readonly conversationId?: string;
+ readonly imageLimit?: number;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
+ ): Promise<readonly ChatMessage[]> {
+ // Fast path: no images anywhere → nothing to do.
+ if (!hasImageChunks(messages)) return messages;
+
+ // Resolve compact URLs (/images/...) → data URLs for the provider.
+ // The persisted chunks store compact URLs (tiny strings); the provider
+ // needs data URLs (read from tmp files at runtime).
+ const resolved = await resolveImageUrlsInMessages(messages);
+
+ const isCapable =
+ currentModelName !== undefined &&
+ (await isVisionCapable(currentModelName, await getInfo(currentModelName)));
+
+ // ── Vision-capable model: image compaction ──────────────────────────
+ // When the conversation has more images than the limit, the oldest images
+ // are transcribed to text (one-time, cached) and stripped from the
+ // provider messages. Recent images (within the limit) stay native.
+ if (isCapable) {
+ return compactImagesForVisionModel(resolved, opts, currentModelName);
+ }
+
+ // ── Non-vision model: placeholders + consult_vision ──────────────────
+ const vision = await resolveVisionModel();
+ const convId = opts?.conversationId;
+
+ const placeholderFn =
+ vision !== undefined && convId !== undefined
+ ? (id: number) => formatImagePlaceholder(id)
+ : () => formatNoVisionPlaceholder();
+
+ // Replace each image chunk with a numbered placeholder. Assign sequential
+ // 1-based IDs across all messages and register each image in the
+ // per-conversation registry so the consult_vision tool can look it up.
+ let seqId = 0;
+ const result: ChatMessage[] = [];
+ for (const msg of resolved) {
+ if (!msg.chunks.some((c) => c.type === "image")) {
+ result.push(msg);
+ continue;
+ }
+ const newChunks: Chunk[] = [];
+ for (const chunk of msg.chunks) {
+ if (chunk.type === "image") {
+ seqId++;
+ if (convId !== undefined && vision !== undefined) {
+ let convImages = imageRegistry.get(convId);
+ if (convImages === undefined) {
+ convImages = new Map();
+ imageRegistry.set(convId, convImages);
+ }
+ convImages.set(seqId, {
+ url: chunk.url,
+ ...(chunk.mimeType !== undefined ? { mimeType: chunk.mimeType } : {}),
+ });
+ }
+ newChunks.push({ type: "text", text: placeholderFn(seqId) });
+ } else {
+ newChunks.push(chunk);
+ }
+ }
+ result.push({ role: msg.role, chunks: newChunks });
+ }
+ return result;
+ },
+
+ getRegisteredImage(conversationId: string, imageId: number): RegisteredImage | undefined {
+ return imageRegistry.get(conversationId)?.get(imageId);
+ },
+
+ async consultVision(
+ question: string,
+ opts: {
+ readonly conversationId: string;
+ readonly imageIds?: readonly number[];
+ readonly path?: string;
+ readonly cwd?: string;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
+ ): Promise<
+ { readonly conversationId: string; readonly response: string } | { readonly error: string }
+ > {
+ const orchestrator = deps.resolveOrchestrator?.();
+ if (orchestrator === undefined) {
+ return {
+ error: "The session orchestrator is not available — cannot start a vision consultation.",
+ };
+ }
+
+ const vision = await resolveVisionModel();
+ if (vision === undefined) {
+ return {
+ error:
+ "No vision-capable model is available in the catalog. Install or configure one (e.g. kimi) to enable image analysis.",
+ };
+ }
+
+ // Collect image data URLs to attach.
+ const images: ImageInput[] = [];
+ if (opts.imageIds !== undefined) {
+ for (const id of opts.imageIds) {
+ const img = service.getRegisteredImage(opts.conversationId, id);
+ if (img === undefined) {
+ return {
+ error: `Image ${id} is not registered. It may have been lost after a server restart — ask the user to re-paste the image.`,
+ };
+ }
+ images.push({
+ url: img.url,
+ ...(img.mimeType !== undefined ? { mimeType: img.mimeType } : {}),
+ });
+ }
+ }
+ if (opts.path !== undefined) {
+ try {
+ const dataUrl = await deps.readFileAsDataUrl(opts.path, opts.cwd);
+ images.push({ url: dataUrl });
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ return { error: `Failed to read image file "${opts.path}": ${msg}` };
+ }
+ }
+ if (images.length === 0) {
+ return {
+ error:
+ "No image to consult about. Provide imageIds (for pasted images) or path (for a file).",
+ };
+ }
+
+ // Start a NEW conversation with the vision model.
+ const consultationId = generateId();
+ log?.info("vision-handoff: starting consultation", {
+ consultationId,
+ visionModel: vision.modelName,
+ imageCount: images.length,
+ fromConversation: opts.conversationId,
+ });
+
+ // Label the consultation tab with an "IMAGE - " prefix so it's visually
+ // distinguishable from normal conversation tabs. Set BEFORE the turn
+ // starts so the tab shows the correct title from the first moment (the
+ // store keeps a non-"Untitled" title on first message append).
+ if (deps.setConversationTitle !== undefined) {
+ try {
+ await deps.setConversationTitle(consultationId, formatConsultationTitle(question));
+ } catch (err) {
+ // Best-effort — don't let a title-write failure break the consultation.
+ log?.warn("vision-handoff: failed to set consultation title", {
+ consultationId,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ }
+ }
+
+ let responseText = "";
+ let errorMessage = "";
+ try {
+ await orchestrator.handleMessage({
+ conversationId: consultationId,
+ text: question,
+ images,
+ modelName: vision.modelName,
+ ...(opts.cwd !== undefined ? { cwd: opts.cwd } : {}),
+ systemPrompt:
+ "You are a vision assistant. A developer who cannot see images is asking you specific " +
+ "questions about an image they attached. Answer their question precisely and thoroughly. " +
+ "Do not use any tools unless specifically asked to — just use your vision to see the " +
+ "image and describe it directly.",
+ onEvent: (event: AgentEvent) => {
+ if (event.type === "text-delta") {
+ responseText += event.delta;
+ } else if (event.type === "error") {
+ errorMessage = event.message;
+ }
+ },
+ });
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ return { error: `Vision consultation failed: ${msg}` };
+ }
+
+ if (errorMessage.length > 0 && responseText.trim().length === 0) {
+ return { error: `Vision consultation failed: ${errorMessage}` };
+ }
+
+ const response = formatConsultResult(consultationId, responseText);
+ return { conversationId: consultationId, response };
+ },
+ };
+
+ return service;
+}
diff --git a/packages/vision-handoff/src/tool.ts b/packages/vision-handoff/src/tool.ts
new file mode 100644
index 0000000..86be2ed
--- /dev/null
+++ b/packages/vision-handoff/src/tool.ts
@@ -0,0 +1,137 @@
+/**
+ * consult_vision tool — lets any model (vision-capable or not) consult a
+ * vision-capable model about an image by opening a NEW conversation tab.
+ *
+ * The tool attaches image(s) + the model's specific question to a vision-capable
+ * model (resolved from the catalog — e.g. Kimi), waits for the response, and
+ * returns the conversation ID + the vision model's answer. The MODEL directs the
+ * analysis — it asks exactly what it needs to know — instead of receiving a
+ * pre-emptive generic dump.
+ *
+ * For images PASTED into the chat, the model references them by `imageIds` (from
+ * the "[Image N attached]" placeholders the orchestrator injected). For image
+ * FILES on disk, the model passes a `path`.
+ *
+ * Follow-up questions are NOT handled by this tool — the model uses the dispatch
+ * CLI to continue the vision conversation (the returned conversation ID is the
+ * bridge; the model can load the `dispatch-cli` skill for the exact commands).
+ */
+
+import type { ToolContract, ToolExecuteContext, ToolResult } from "@dispatch/kernel";
+import type { VisionHandoffService } from "./service.js";
+
+export function createConsultVisionTool(service: VisionHandoffService): ToolContract {
+ return {
+ name: "consult_vision",
+ description:
+ "Consult a vision-capable model (e.g. Kimi) about an image by opening a new " +
+ "conversation tab. Attaches the image(s) + your specific question, waits for " +
+ "the vision model's response, and returns the conversation ID + the answer. " +
+ "Use this when you cannot view an image (e.g. a pasted screenshot or diagram) " +
+ "and need to know what it shows — ask a SPECIFIC question (e.g. 'What error " +
+ "message is on line 12?' rather than 'describe this image'). The conversation " +
+ "ID is returned so follow-up questions can be asked via the dispatch CLI.",
+ parameters: {
+ type: "object",
+ properties: {
+ question: {
+ type: "string",
+ description:
+ "Your specific question about the image. Be precise — the vision model " +
+ "will answer exactly this. E.g. 'What error message is displayed?' or " +
+ "'Compare the layout of these two screenshots.'",
+ },
+ imageIds: {
+ type: "array",
+ items: { type: "number" },
+ description:
+ "The IDs of pasted images to attach (from the '[Image N attached]' " +
+ "placeholders in the conversation). Pass multiple to attach several " +
+ "images to one consultation (e.g. [1, 2] to compare them).",
+ },
+ path: {
+ type: "string",
+ description:
+ "Path to an image FILE on disk to attach (alternative to imageIds for " +
+ "code-referenced images). Relative paths resolve against the cwd.",
+ },
+ },
+ required: ["question"],
+ },
+ concurrencySafe: true,
+ async execute(args: unknown, ctx: ToolExecuteContext): Promise<ToolResult> {
+ const input = args as {
+ question?: unknown;
+ imageIds?: unknown;
+ path?: unknown;
+ } | null;
+
+ const question = input?.question;
+ if (typeof question !== "string" || question.trim().length === 0) {
+ return {
+ content: "Error: 'question' is required and must be a non-empty string.",
+ isError: true,
+ };
+ }
+
+ const imageIds = input?.imageIds;
+ const path = input?.path;
+
+ // Parse imageIds (must be an array of numbers if present).
+ let parsedImageIds: number[] | undefined;
+ if (imageIds !== undefined) {
+ if (!Array.isArray(imageIds)) {
+ return { content: "Error: 'imageIds' must be an array of numbers.", isError: true };
+ }
+ parsedImageIds = imageIds.filter((n): n is number => typeof n === "number");
+ if (parsedImageIds.length === 0) {
+ return { content: "Error: 'imageIds' must contain at least one number.", isError: true };
+ }
+ }
+
+ // path must be a string if present.
+ let parsedPath: string | undefined;
+ if (path !== undefined) {
+ if (typeof path !== "string" || path.trim().length === 0) {
+ return { content: "Error: 'path' must be a non-empty string.", isError: true };
+ }
+ parsedPath = path;
+ }
+
+ // At least one image source is required.
+ if (parsedImageIds === undefined && parsedPath === undefined) {
+ return {
+ content:
+ "Error: provide 'imageIds' (for pasted images) or 'path' (for a file) " +
+ "to attach an image to the consultation.",
+ isError: true,
+ };
+ }
+
+ const span = ctx.log.span("consult_vision.execute", {
+ imageCount: (parsedImageIds?.length ?? 0) + (parsedPath !== undefined ? 1 : 0),
+ });
+ try {
+ const result = await service.consultVision(question, {
+ conversationId: ctx.conversationId ?? "",
+ ...(parsedImageIds !== undefined ? { imageIds: parsedImageIds } : {}),
+ ...(parsedPath !== undefined ? { path: parsedPath } : {}),
+ ...(ctx.cwd !== undefined ? { cwd: ctx.cwd } : {}),
+ signal: ctx.signal,
+ logger: ctx.log,
+ });
+ span.end({ attrs: { ok: !("error" in result) } });
+ if ("error" in result) {
+ return { content: result.error, isError: true };
+ }
+ return { content: result.response };
+ } catch (err: unknown) {
+ span.end({ err });
+ return {
+ content: `Error during vision consultation: ${err instanceof Error ? err.message : String(err)}`,
+ isError: true,
+ };
+ }
+ },
+ };
+}
diff --git a/packages/vision-handoff/tsconfig.json b/packages/vision-handoff/tsconfig.json
new file mode 100644
index 0000000..b5439aa
--- /dev/null
+++ b/packages/vision-handoff/tsconfig.json
@@ -0,0 +1,12 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": { "rootDir": "src", "outDir": "dist", "composite": true },
+ "include": ["src/**/*.ts"],
+ "references": [
+ { "path": "../kernel" },
+ { "path": "../wire" },
+ { "path": "../conversation-store" },
+ { "path": "../credential-store" },
+ { "path": "../openai-stream" }
+ ]
+}