diff options
Diffstat (limited to 'packages/vision-handoff/src')
| -rw-r--r-- | packages/vision-handoff/src/extension.ts | 198 | ||||
| -rw-r--r-- | packages/vision-handoff/src/index.ts | 21 | ||||
| -rw-r--r-- | packages/vision-handoff/src/pure.test.ts | 180 | ||||
| -rw-r--r-- | packages/vision-handoff/src/pure.ts | 156 | ||||
| -rw-r--r-- | packages/vision-handoff/src/service.test.ts | 375 | ||||
| -rw-r--r-- | packages/vision-handoff/src/service.ts | 689 | ||||
| -rw-r--r-- | packages/vision-handoff/src/tool.ts | 137 |
7 files changed, 1756 insertions, 0 deletions
diff --git a/packages/vision-handoff/src/extension.ts b/packages/vision-handoff/src/extension.ts new file mode 100644 index 0000000..08fddca --- /dev/null +++ b/packages/vision-handoff/src/extension.ts @@ -0,0 +1,198 @@ +/** + * vision-handoff extension — registers the universal vision handoff service + + * the `consult_vision` tool. + * + * The service performs provider-agnostic vision handoff: when a non-vision model + * (e.g. glm-5.2) receives an image, it replaces the image with a numbered + * placeholder and registers it for tool access. The `consult_vision` tool opens + * a NEW conversation tab with a vision-capable model (e.g. Kimi), attaches the + * image + the model's specific question, and returns the conversation ID + the + * vision model's answer. Follow-ups go through the dispatch CLI. + * + * Images are saved to a tmp directory (`/tmp/dispatch/images/<convId>/`) so the + * conversation store (SQLite) only holds a compact URL reference — not + * megabytes of base64. Tmp files are purged on reboot (ephemeral dir), after + * compaction (the transcription replaces the image), and on conversation close. + * + * Effects (filesystem, orchestrator) live here in the shell, injected into the + * service. The pure decisions live in `pure.ts`. No `console.*`; logging via + * `host.logger`. + */ + +import { mkdir, readFile, rm, unlink, writeFile } from "node:fs/promises"; +import { extname, isAbsolute, join, resolve as pathResolve } from "node:path"; +import { conversationStoreHandle } from "@dispatch/conversation-store"; +import type { CredentialStore } from "@dispatch/credential-store"; +import { credentialStoreHandle } from "@dispatch/credential-store"; +import type { Extension, HostAPI, Manifest } from "@dispatch/kernel"; +import { + createVisionHandoffService, + orchestratorLocalHandle, + visionHandoffHandle, +} from "./service.js"; +import { createConsultVisionTool } from "./tool.js"; + +export const manifest: Manifest = { + id: "vision-handoff", + name: "Vision Handoff", + version: "0.0.0", + apiVersion: "^0.1.0", + trust: "bundled", + activation: "eager", + capabilities: { network: true }, + contributes: { services: ["vision-handoff/service"], tools: ["consult_vision"] }, +}; + +const IMAGE_DIR = process.env.DISPATCH_IMAGE_DIR ?? "/tmp/dispatch/images"; + +/** MIME types for recognized image extensions. */ +const MIME_BY_EXT: Readonly<Record<string, string>> = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".webp": "image/webp", + ".gif": "image/gif", + ".bmp": "image/bmp", +}; + +/** Reverse: MIME → extension. */ +const EXT_BY_MIME: Readonly<Record<string, string>> = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/webp": ".webp", + "image/gif": ".gif", + "image/bmp": ".bmp", +}; + +/** + * Read an image file from disk as a base64 data URL. Resolves relative paths + * against the cwd (the conversation's working directory). Throws on missing + * file / read error (the caller surfaces it). The shell edge — real `node:fs`. + */ +async function readFileAsDataUrl(path: string, cwd?: string): Promise<string> { + const abs = cwd !== undefined && !isAbsolute(path) ? pathResolve(cwd, path) : pathResolve(path); + const buf = await readFile(abs); + const ext = extname(abs).toLowerCase(); + const mime = MIME_BY_EXT[ext] ?? "image/png"; + return `data:${mime};base64,${buf.toString("base64")}`; +} + +/** + * Save a data URL image to a tmp file and return a compact HTTP path. + * The compact URL (`/images/<conversationId>/<uuid>.<ext>`) is what gets + * persisted in the conversation store — a tiny string, not megabytes of base64. + */ +async function saveImageToTmp( + conversationId: string, + dataUrl: string, + mimeType?: string, +): Promise<string> { + const mime = mimeType ?? "image/png"; + const ext = EXT_BY_MIME[mime] ?? ".png"; + const imageId = `${crypto.randomUUID()}${ext}`; + const dir = join(IMAGE_DIR, conversationId); + await mkdir(dir, { recursive: true }); + const filePath = join(dir, imageId); + const base64 = dataUrl.split(",")[1] ?? ""; + await writeFile(filePath, Buffer.from(base64, "base64")); + return `/images/${conversationId}/${imageId}`; +} + +/** + * Resolve a compact URL (`/images/<convId>/<imageId>`) back to a data URL by + * reading the tmp file. Data URLs and HTTP URLs pass through unchanged. + */ +async function resolveImageUrl(url: string): Promise<string> { + if (url.startsWith("data:") || url.startsWith("http")) return url; + if (!url.startsWith("/images/")) return url; + const parts = url.split("/"); // ["", "images", convId, imageId] + const convId = parts[2]; + const imageId = parts[3]; + if (convId === undefined || imageId === undefined) return url; + const filePath = join(IMAGE_DIR, convId, imageId); + const buf = await readFile(filePath); + const ext = extname(imageId).toLowerCase(); + const mime = MIME_BY_EXT[ext] ?? "image/png"; + return `data:${mime};base64,${buf.toString("base64")}`; +} + +/** Delete a single tmp image file (after compaction — best-effort). */ +async function deleteTmpImage(compactUrl: string): Promise<void> { + if (!compactUrl.startsWith("/images/")) return; + const parts = compactUrl.split("/"); + const convId = parts[2]; + const imageId = parts[3]; + if (convId === undefined || imageId === undefined) return; + const filePath = join(IMAGE_DIR, convId, imageId); + try { + await unlink(filePath); + } catch { + // Best-effort — file may already be deleted. + } +} + +/** Delete all tmp images for a conversation (on close — best-effort). */ +async function deleteConversationImages(conversationId: string): Promise<void> { + const dir = join(IMAGE_DIR, conversationId); + try { + await rm(dir, { recursive: true, force: true }); + } catch { + // Best-effort. + } +} + +export async function activate(host: HostAPI): Promise<void> { + const credentialStore = host.getService(credentialStoreHandle) as CredentialStore | undefined; + if (credentialStore === undefined) { + host.logger.warn( + "vision-handoff: credential-store service not available. The consult_vision tool and image handoff are disabled.", + ); + return; + } + + const resolveModel = (modelName: string) => { + const resolved = credentialStore.resolve(modelName); + if (resolved === undefined) return undefined; + const provider = host.getProviders().get(resolved.providerId); + if (provider === undefined) return undefined; + return { provider, model: resolved.model }; + }; + + const service = createVisionHandoffService({ + credentialStore, + resolveModel, + readFileAsDataUrl, + saveImageToTmp, + resolveImageUrl, + deleteTmpImage, + deleteConversationImages, + resolveOrchestrator: () => { + const loaded = host.getExtensions().some((m) => m.id === "session-orchestrator"); + if (!loaded) return undefined; + try { + return host.getService(orchestratorLocalHandle); + } catch { + return undefined; + } + }, + getImageTranscriptions: async (conversationId: string) => { + const store = host.getService(conversationStoreHandle); + return store.getImageTranscriptions(conversationId); + }, + setImageTranscription: async (conversationId: string, url: string, text: string) => { + const store = host.getService(conversationStoreHandle); + await store.setImageTranscription(conversationId, url, text); + }, + setConversationTitle: async (conversationId: string, title: string) => { + const store = host.getService(conversationStoreHandle); + await store.setConversationTitle(conversationId, title); + }, + logger: host.logger.child({ extensionId: "vision-handoff" }), + }); + + host.provideService(visionHandoffHandle, service); + host.defineTool(createConsultVisionTool(service)); + host.logger.info("vision-handoff: registered (consult_vision tool + handoff service)"); +} + +export const extension: Extension = { manifest, activate }; diff --git a/packages/vision-handoff/src/index.ts b/packages/vision-handoff/src/index.ts new file mode 100644 index 0000000..2713346 --- /dev/null +++ b/packages/vision-handoff/src/index.ts @@ -0,0 +1,21 @@ +export { extension, manifest } from "./extension.js"; +export { + collectTextFromStream, + findVisionModelName, + formatConsultResult, + formatImagePlaceholder, + formatNoVisionPlaceholder, + isVisionCapable, +} from "./pure.js"; +export type { + OrchestratorForVision, + ResolvedVisionModel, + VisionHandoffDeps, + VisionHandoffService, +} from "./service.js"; +export { + createVisionHandoffService, + orchestratorLocalHandle, + visionHandoffHandle, +} from "./service.js"; +export { createConsultVisionTool } from "./tool.js"; diff --git a/packages/vision-handoff/src/pure.test.ts b/packages/vision-handoff/src/pure.test.ts new file mode 100644 index 0000000..21b1224 --- /dev/null +++ b/packages/vision-handoff/src/pure.test.ts @@ -0,0 +1,180 @@ +import type { ModelInfo, ProviderEvent } from "@dispatch/kernel"; +import { describe, expect, it } from "vitest"; +import { + collectTextFromStream, + findVisionModelName, + formatConsultationTitle, + formatConsultResult, + formatImagePlaceholder, + formatNoVisionPlaceholder, + isVisionCapable, +} from "./pure.js"; + +describe("isVisionCapable", () => { + it("returns true when ModelInfo.vision is true", () => { + expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: true })).toBe( + true, + ); + }); + + it("returns false when ModelInfo.vision is false (overrides name heuristic)", () => { + expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: false })).toBe( + false, + ); + }); + + it("falls back to name heuristic when vision is absent (umans kimi + qwen)", () => { + expect(isVisionCapable("umans/umans-kimi-k2.7", undefined)).toBe(true); + expect(isVisionCapable("umans/umans-qwen3.6-35b-a3b", undefined)).toBe(true); + }); + + it("falls back to name heuristic when vision is absent (non-vision)", () => { + expect(isVisionCapable("umans/umans-glm-5.2", undefined)).toBe(false); + expect(isVisionCapable("umans/umans-coder", { id: "umans-coder" })).toBe(false); + }); + + it("returns false for undefined model name", () => { + expect(isVisionCapable(undefined, undefined)).toBe(false); + }); +}); + +describe("findVisionModelName", () => { + const getInfo = async (name: string): Promise<ModelInfo | undefined> => { + const map: Record<string, ModelInfo> = { + "umans/umans-kimi-k2.7": { id: "umans-kimi-k2.7", vision: true }, + "umans/umans-qwen3.6-35b-a3b": { id: "umans-qwen3.6-35b-a3b", vision: true }, + "umans/umans-glm-5.2": { id: "umans-glm-5.2" }, + "umans/llama-vision": { id: "llama-vision", vision: true }, + }; + return map[name]; + }; + + it("finds the first umans kimi model via name heuristic", async () => { + const name = await findVisionModelName( + ["umans/umans-glm-5.2", "umans/umans-kimi-k2.7", "umans/llama-vision"], + getInfo, + ); + expect(name).toBe("umans/umans-kimi-k2.7"); + }); + + it("finds a vision model via ModelInfo.vision when name heuristic misses", async () => { + const name = await findVisionModelName(["umans/umans-glm-5.2", "umans/llama-vision"], getInfo); + expect(name).toBe("umans/llama-vision"); + }); + + it("skips the excluded model and finds the next vision model", async () => { + const name = await findVisionModelName( + ["umans/umans-kimi-k2.7", "umans/umans-qwen3.6-35b-a3b"], + getInfo, + "umans/umans-kimi-k2.7", + ); + expect(name).toBe("umans/umans-qwen3.6-35b-a3b"); + }); + + it("returns undefined when no vision model is available", async () => { + const name = await findVisionModelName(["umans/umans-glm-5.2"], getInfo); + expect(name).toBeUndefined(); + }); + + it("returns undefined for empty catalog", async () => { + const name = await findVisionModelName([], getInfo); + expect(name).toBeUndefined(); + }); +}); + +describe("collectTextFromStream", () => { + async function* stream(events: ProviderEvent[]): AsyncIterable<ProviderEvent> { + for (const e of events) yield e; + } + + it("collects text-delta events into a single string", async () => { + const events: ProviderEvent[] = [ + { type: "text-delta", delta: "Hello " }, + { type: "text-delta", delta: "world!" }, + ]; + const text = await collectTextFromStream(stream(events)); + expect(text).toBe("Hello world!"); + }); + + it("ignores non-text events", async () => { + const events: ProviderEvent[] = [ + { type: "reasoning-delta", delta: "thinking..." }, + { type: "text-delta", delta: "answer" }, + { type: "usage", usage: { inputTokens: 5, outputTokens: 1 } }, + { type: "finish", reason: "stop" }, + ]; + const text = await collectTextFromStream(stream(events)); + expect(text).toBe("answer"); + }); + + it("throws on an error event", async () => { + const events: ProviderEvent[] = [ + { type: "text-delta", delta: "partial" }, + { type: "error", message: "boom" }, + ]; + await expect(collectTextFromStream(stream(events))).rejects.toThrow("boom"); + }); + + it("returns empty string for an empty stream", async () => { + const text = await collectTextFromStream(stream([])); + expect(text).toBe(""); + }); +}); + +describe("formatImagePlaceholder", () => { + it("includes the image ID and mentions consult_vision", () => { + const text = formatImagePlaceholder(1); + expect(text).toContain("Image 1"); + expect(text).toContain("consult_vision"); + expect(text).toContain("imageIds=[1]"); + }); + + it("increments the ID for each image", () => { + expect(formatImagePlaceholder(2)).toContain("Image 2"); + expect(formatImagePlaceholder(2)).toContain("imageIds=[2]"); + }); +}); + +describe("formatNoVisionPlaceholder", () => { + it("explains the limitation", () => { + const text = formatNoVisionPlaceholder(); + expect(text).toContain("no vision-capable model"); + }); +}); + +describe("formatConsultResult", () => { + it("includes the conversation ID, the response, and the dispatch CLI hint", () => { + const result = formatConsultResult("abc-123", "The error is on line 12."); + expect(result).toContain("abc-123"); + expect(result).toContain("The error is on line 12."); + expect(result).toContain("dispatch CLI"); + }); + + it("trims the response", () => { + const result = formatConsultResult("c1", " spaced "); + expect(result).toContain("spaced"); + expect(result).not.toContain("spaced "); + }); +}); + +describe("formatConsultationTitle", () => { + it("prefixes the question with 'IMAGE - '", () => { + expect(formatConsultationTitle("What error is shown?")).toBe("IMAGE - What error is shown?"); + }); + + it("truncates long questions to 80 chars with an ellipsis (matching the store's TITLE_MAX)", () => { + const long = "x".repeat(100); + const title = formatConsultationTitle(long); + expect(title).toBe(`IMAGE - ${"x".repeat(80)}…`); + expect(title.length).toBe("IMAGE - ".length + 80 + 1); // prefix + 80 + ellipsis + }); + + it("does not truncate questions at or under 80 chars", () => { + expect(formatConsultationTitle("x".repeat(80))).toBe(`IMAGE - ${"x".repeat(80)}`); + expect(formatConsultationTitle("x".repeat(79))).toBe(`IMAGE - ${"x".repeat(79)}`); + }); + + it("handles an empty question", () => { + expect(formatConsultationTitle("")).toBe("IMAGE - "); + }); +}); diff --git a/packages/vision-handoff/src/pure.ts b/packages/vision-handoff/src/pure.ts new file mode 100644 index 0000000..af3476f --- /dev/null +++ b/packages/vision-handoff/src/pure.ts @@ -0,0 +1,156 @@ +/** + * Pure decision helpers for the vision handoff. + * + * No I/O, no ambient state. The shell (the extension + the service) injects the + * effects (credential store lookups, orchestrator, provider streaming). This + * module owns only the policy: which model is vision-capable, how to format + * image placeholders for non-vision models, and how to format the + * consultation tool's result. + */ + +import type { ModelInfo, ProviderEvent } from "@dispatch/kernel"; +import { isVisionModelId } from "@dispatch/openai-stream"; + +/** + * Whether a model is vision-capable, given its catalog name and (optional) + * resolved `ModelInfo`. When `ModelInfo.vision` is present it is authoritative; + * otherwise fall back to the hardcoded name heuristic ({@link isVisionModelId}). + * + * The `modelName` is the `<credentialName>/<model>` catalog form; the heuristic + * inspects the model SEGMENT (after the first `/`) so `umans/kimi-k2.7` → the + * `kimi-k2.7` segment is checked. Pure. + */ +export function isVisionCapable( + modelName: string | undefined, + info: ModelInfo | undefined, +): boolean { + // When ModelInfo explicitly reports vision (true OR false), it is authoritative + // — an explicit false overrides the name heuristic (a provider that KNOWS a + // model is non-vision wins over the name guess). + if (info?.vision !== undefined) return info.vision; + if (modelName === undefined) return false; + const slash = modelName.indexOf("/"); + const modelId = slash >= 0 ? modelName.slice(slash + 1) : modelName; + return isVisionModelId(modelId); +} + +/** + * Find the first vision-capable model name in a catalog, given a lookup that + * resolves a `<credentialName>/<model>` → `ModelInfo`. Returns `undefined` when + * no vision-capable model is available. Pure given the (async) lookup. + * + * @param catalog The full list of model names (`<credentialName>/<model>`). + * @param getInfo Async lookup of a model name → ModelInfo (from the credential store). + * @param exclude Optional model name to skip (e.g. the current non-vision model). + */ +export async function findVisionModelName( + catalog: readonly string[], + getInfo: (modelName: string) => Promise<ModelInfo | undefined>, + exclude?: string, +): Promise<string | undefined> { + for (const name of catalog) { + if (exclude !== undefined && name === exclude) continue; + // Fast path: the name heuristic lets us short-circuit without an async + // lookup for known vision families (kimi). + const slash = name.indexOf("/"); + const modelId = slash >= 0 ? name.slice(slash + 1) : name; + if (isVisionModelId(modelId)) return name; + const info = await getInfo(name); + if (info?.vision === true) return name; + } + return undefined; +} + +/** + * Fold a provider's streamed events into a single text string. Pure given the + * async iterable — collects `text-delta` events, ignores everything else + * (reasoning, usage, tool-calls). If the stream yields an error event, it is + * surfaced as a thrown Error so the caller can decide how to degrade. + */ +export async function collectTextFromStream(stream: AsyncIterable<ProviderEvent>): Promise<string> { + let text = ""; + for await (const event of stream) { + if (event.type === "text-delta") { + text += event.delta; + } else if (event.type === "error") { + throw new Error(event.message); + } + } + return text; +} + +/** + * Format the placeholder text that replaces an `image` chunk when a non-vision + * model is active. The placeholder tells the model an image is attached and it + * should call `consult_vision` to analyze it — the model drives the analysis + * (asking a specific question) rather than receiving a pre-emptive generic dump. + * + * @param imageId The 1-based ID assigned to this image (used by the tool to + * look up the registered image data). + * Pure. + */ +export function formatImagePlaceholder(imageId: number): string { + return ( + `[Image ${imageId} attached — you cannot view images. Call the ` + + `consult_vision tool with imageIds=[${imageId}] and a specific question ` + + `to analyze it via a vision-capable model.]` + ); +} + +/** + * Placeholder text used when NO vision-capable model is available (the + * degraded path — the tool cannot function). Pure. + */ +export function formatNoVisionPlaceholder(): string { + return ( + "[Image attached — no vision-capable model is available to analyze it. " + + "Install or configure a vision-capable model (e.g. kimi) to enable image analysis.]" + ); +} + +/** + * Maximum length of the consultation title body (matching the conversation + * store's `TITLE_MAX`). The question is truncated to this before the + * `"IMAGE - "` prefix is applied so the consultation tab's title stays in line + * with the store's own title-derivation limit. + */ +const CONSULTATION_TITLE_MAX = 80; + +/** + * Format the title for a vision consultation conversation tab. The title is + * `"IMAGE - "` prefixed to the (truncated) question so the tab is visually + * distinguishable from normal conversation tabs. The question is truncated to + * match the conversation store's title-derivation limit (`TITLE_MAX = 80`). + * + * Pure. + * + * @param question The question the model asked the vision model. + */ +export function formatConsultationTitle(question: string): string { + const body = + question.length > CONSULTATION_TITLE_MAX + ? `${question.slice(0, CONSULTATION_TITLE_MAX)}…` + : question; + return `IMAGE - ${body}`; +} + +/** + * Format the `consult_vision` tool's result string. Returns the conversation ID + * (so the model / user can continue the vision consultation), the vision model's + * response, and a note that follow-up questions use the dispatch CLI (the model + * can load the `dispatch-cli` skill for the exact commands). + * + * Pure. + * + * @param conversationId The new vision consultation conversation ID. + * @param response The vision model's answer to the model's question. + */ +export function formatConsultResult(conversationId: string, response: string): string { + const trimmed = response.trim(); + return ( + `Vision consultation opened in conversation ${conversationId}.\n\n` + + `Response: ${trimmed}\n\n` + + `To ask follow-up questions about this image, use the dispatch CLI ` + + `(conversation: ${conversationId}).` + ); +} diff --git a/packages/vision-handoff/src/service.test.ts b/packages/vision-handoff/src/service.test.ts new file mode 100644 index 0000000..8c4117e --- /dev/null +++ b/packages/vision-handoff/src/service.test.ts @@ -0,0 +1,375 @@ +import type { + AgentEvent, + ChatMessage, + ModelInfo, + ProviderContract, + ProviderEvent, + ToolContract, +} from "@dispatch/kernel"; +import { describe, expect, it, vi } from "vitest"; +import { createVisionHandoffService, type VisionHandoffDeps } from "./service.js"; + +// ── Test doubles (outermost-edge fakes — NOT @dispatch/* mocks) ────────────── + +function makeVisionProvider( + describe: (imageUrl: string) => string, + id = "umans", +): ProviderContract { + return { + id, + stream: vi.fn( + ( + messages: readonly ChatMessage[], + _tools: readonly ToolContract[], + ): AsyncIterable<ProviderEvent> => { + const img = messages.flatMap((m) => m.chunks).find((c) => c.type === "image"); + const url = img && img.type === "image" ? img.url : ""; + const text = describe(url); + async function* gen(): AsyncIterable<ProviderEvent> { + yield { type: "text-delta", delta: text }; + yield { type: "finish", reason: "stop" }; + } + return gen(); + }, + ), + }; +} + +function makeDeps(overrides: Partial<VisionHandoffDeps> = {}): VisionHandoffDeps { + const visionProvider = makeVisionProvider((url) => `DESCRIPTION of ${url}`); + const catalog = ["umans/umans-kimi-k2.7", "umans/umans-glm-5.2"]; + const infoMap: Record<string, ModelInfo> = { + "umans/umans-kimi-k2.7": { id: "umans-kimi-k2.7", vision: true }, + "umans/umans-glm-5.2": { id: "umans-glm-5.2" }, + }; + return { + credentialStore: { + listCatalog: vi.fn(async () => catalog), + getModelInfo: vi.fn(async (name: string) => infoMap[name]), + resolve: vi.fn((name: string) => { + if (name === "umans/umans-kimi-k2.7") + return { providerId: "umans", model: "umans-kimi-k2.7" }; + if (name === "umans/umans-glm-5.2") return { providerId: "umans", model: "umans-glm-5.2" }; + return undefined; + }), + }, + resolveModel: vi.fn((name: string) => + name === "umans/umans-kimi-k2.7" || name === "umans/umans-glm-5.2" + ? { provider: visionProvider, model: name.split("/")[1] } + : undefined, + ), + readFileAsDataUrl: vi.fn(async (path: string) => `data:image/png;base64,FILE(${path})`), + setConversationTitle: vi.fn(async (_conversationId: string, _title: string) => {}), + ...overrides, + }; +} + +describe("VisionHandoffService.isVisionCapable", () => { + it("returns true for kimi (via ModelInfo)", async () => { + const svc = createVisionHandoffService(makeDeps()); + expect(await svc.isVisionCapable("umans/umans-kimi-k2.7")).toBe(true); + }); + + it("returns false for glm-5.2", async () => { + const svc = createVisionHandoffService(makeDeps()); + expect(await svc.isVisionCapable("umans/umans-glm-5.2")).toBe(false); + }); + + it("returns false for undefined model name", async () => { + const svc = createVisionHandoffService(makeDeps()); + expect(await svc.isVisionCapable(undefined)).toBe(false); + }); +}); + +describe("VisionHandoffService.resolveVisionModel", () => { + it("resolves the kimi model from the catalog", async () => { + const svc = createVisionHandoffService(makeDeps()); + const vision = await svc.resolveVisionModel(); + expect(vision?.modelName).toBe("umans/umans-kimi-k2.7"); + expect(vision?.model).toBe("umans-kimi-k2.7"); + }); + + it("excludes the given model", async () => { + const svc = createVisionHandoffService(makeDeps()); + const vision = await svc.resolveVisionModel("umans/umans-kimi-k2.7"); + expect(vision).toBeUndefined(); + }); +}); + +describe("VisionHandoffService.prepareForProvider", () => { + it("passes messages through unchanged when the model is vision-capable", async () => { + const deps = makeDeps(); + const svc = createVisionHandoffService(deps); + const messages: ChatMessage[] = [ + { + role: "user", + chunks: [ + { type: "text", text: "What's this?" }, + { type: "image", url: "data:image/png;base64,abc" }, + ], + }, + ]; + const result = await svc.prepareForProvider(messages, "umans/umans-kimi-k2.7"); + expect(result).toBe(messages); // same reference — no copy, no change + }); + + it("passes messages through unchanged when there are no images", async () => { + const deps = makeDeps(); + const svc = createVisionHandoffService(deps); + const messages: ChatMessage[] = [{ role: "user", chunks: [{ type: "text", text: "hi" }] }]; + const result = await svc.prepareForProvider(messages, "umans/umans-glm-5.2"); + expect(result).toBe(messages); + }); + + it("replaces image chunks with numbered placeholders for a non-vision model", async () => { + const deps = makeDeps(); + const svc = createVisionHandoffService(deps); + const messages: ChatMessage[] = [ + { + role: "user", + chunks: [ + { type: "text", text: "Describe this" }, + { type: "image", url: "data:image/png;base64,img1" }, + ], + }, + ]; + const result = await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { + conversationId: "conv-1", + }); + expect(result).toHaveLength(1); + const chunks = result[0]?.chunks; + expect(chunks).toHaveLength(2); + // Text chunk unchanged. + expect(chunks?.[0]).toEqual({ type: "text", text: "Describe this" }); + // Image chunk → placeholder text. + expect(chunks?.[1]?.type).toBe("text"); + const placeholder = (chunks?.[1] as { text: string }).text; + expect(placeholder).toContain("Image 1"); + expect(placeholder).toContain("consult_vision"); + }); + + it("assigns sequential image IDs across multiple messages", async () => { + const deps = makeDeps(); + const svc = createVisionHandoffService(deps); + const messages: ChatMessage[] = [ + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,a" }] }, + { role: "assistant", chunks: [{ type: "text", text: "ok" }] }, + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,b" }] }, + ]; + const result = await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { + conversationId: "conv-1", + }); + // First image → Image 1, second → Image 2. + expect((result[0]?.chunks[0] as { text: string }).text).toContain("Image 1"); + // Assistant message unchanged. + expect(result[1]?.chunks[0]?.type).toBe("text"); + expect((result[2]?.chunks[0] as { text: string }).text).toContain("Image 2"); + }); + + it("registers images so getRegisteredImage can look them up", async () => { + const deps = makeDeps(); + const svc = createVisionHandoffService(deps); + const messages: ChatMessage[] = [ + { + role: "user", + chunks: [{ type: "image", url: "data:image/png;base64,registered" }], + }, + ]; + await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { conversationId: "conv-42" }); + const img = svc.getRegisteredImage("conv-42", 1); + expect(img?.url).toBe("data:image/png;base64,registered"); + }); + + it("uses no-vision placeholder when no vision model is available", async () => { + const deps = makeDeps(); + (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]); + const svc = createVisionHandoffService(deps); + const messages: ChatMessage[] = [ + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,abc" }] }, + ]; + const result = await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { + conversationId: "conv-1", + }); + const text = (result[0]?.chunks[0] as { text: string }).text; + expect(text).toContain("no vision-capable model"); + expect(text).not.toContain("consult_vision"); + }); +}); + +describe("VisionHandoffService.consultVision", () => { + function makeOrchestratorDouble(response: string): { + orchestrator: NonNullable< + VisionHandoffDeps["resolveOrchestrator"] extends () => infer T ? T : never + >; + handleMessage: ReturnType<typeof vi.fn>; + } { + const handleMessage = vi.fn( + async (input: { + conversationId: string; + text: string; + onEvent: (event: AgentEvent) => void; + }): Promise<void> => { + input.onEvent({ + type: "text-delta", + conversationId: input.conversationId, + turnId: "t1", + delta: response, + }); + input.onEvent({ + type: "done", + conversationId: input.conversationId, + turnId: "t1", + reason: "stop", + }); + }, + ); + return { orchestrator: { handleMessage }, handleMessage }; + } + + it("opens a new consultation with a pasted image and returns convId + response", async () => { + const deps = makeDeps(); + const { orchestrator, handleMessage } = makeOrchestratorDouble("The error is on line 12."); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + + // Register an image first (as prepareForProvider would). + const messages: ChatMessage[] = [ + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,img1" }] }, + ]; + await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { conversationId: "conv-1" }); + + const result = await svc.consultVision("What error is shown?", { + conversationId: "conv-1", + imageIds: [1], + }); + + expect("error" in result).toBe(false); + if (!("error" in result)) { + expect(result.conversationId).toBeTruthy(); + expect(result.response).toContain("line 12"); + expect(result.response).toContain(result.conversationId); + expect(result.response).toContain("dispatch CLI"); + } + // The orchestrator was called with the vision model + the image. + expect(handleMessage).toHaveBeenCalledOnce(); + const call = handleMessage.mock.calls[0]?.[0]; + expect(call.modelName).toBe("umans/umans-kimi-k2.7"); + expect(call.images).toHaveLength(1); + expect(call.images?.[0]?.url).toBe("data:image/png;base64,img1"); + }); + + it("labels the consultation tab with an 'IMAGE - ' prefixed title", async () => { + const deps = makeDeps(); + const { orchestrator } = makeOrchestratorDouble("The error is on line 12."); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + + // Register an image first (as prepareForProvider would). + const messages: ChatMessage[] = [ + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,img1" }] }, + ]; + await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { conversationId: "conv-1" }); + + const result = await svc.consultVision("What error is shown?", { + conversationId: "conv-1", + imageIds: [1], + }); + + expect("error" in result).toBe(false); + // The title was set with the IMAGE - prefix + the question. + expect(deps.setConversationTitle).toHaveBeenCalledOnce(); + const [titleConvId, title] = (deps.setConversationTitle as ReturnType<typeof vi.fn>).mock + .calls[0]; + expect(titleConvId).toBe((result as { conversationId: string }).conversationId); + expect(title).toBe("IMAGE - What error is shown?"); + }); + + it("does not call setConversationTitle when it is not provided", async () => { + const deps = makeDeps({ setConversationTitle: undefined }); + const { orchestrator } = makeOrchestratorDouble("response"); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + + const messages: ChatMessage[] = [ + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,img1" }] }, + ]; + await svc.prepareForProvider(messages, "umans/umans-glm-5.2", { conversationId: "conv-1" }); + + // Should NOT throw — setConversationTitle is optional. + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + imageIds: [1], + }); + expect("error" in result).toBe(false); + }); + + it("opens a consultation with a file path image", async () => { + const deps = makeDeps(); + const { orchestrator } = makeOrchestratorDouble("It's a diagram."); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + + const result = await svc.consultVision("What is this diagram?", { + conversationId: "conv-1", + path: "diagram.png", + cwd: "/work", + }); + + expect("error" in result).toBe(false); + expect(deps.readFileAsDataUrl).toHaveBeenCalledWith("diagram.png", "/work"); + }); + + it("returns an error when imageId is not registered", async () => { + const deps = makeDeps(); + const { orchestrator } = makeOrchestratorDouble("response"); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + imageIds: [99], // not registered + }); + expect("error" in result).toBe(true); + if ("error" in result) { + expect(result.error).toContain("Image 99"); + } + }); + + it("returns an error when no orchestrator is available", async () => { + const deps = makeDeps(); + // No resolveOrchestrator provided. + const svc = createVisionHandoffService(deps); + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + imageIds: [1], + }); + expect("error" in result).toBe(true); + }); + + it("returns an error when no vision model is available", async () => { + const deps = makeDeps(); + (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]); + const { orchestrator } = makeOrchestratorDouble("response"); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + imageIds: [1], + }); + expect("error" in result).toBe(true); + if ("error" in result) { + expect(result.error).toContain("No vision-capable model"); + } + }); + + it("returns an error when no image source is provided", async () => { + const deps = makeDeps(); + const { orchestrator } = makeOrchestratorDouble("response"); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + }); + expect("error" in result).toBe(true); + }); +}); diff --git a/packages/vision-handoff/src/service.ts b/packages/vision-handoff/src/service.ts new file mode 100644 index 0000000..397d81a --- /dev/null +++ b/packages/vision-handoff/src/service.ts @@ -0,0 +1,689 @@ +/** + * Vision handoff service — the imperative shell that performs the universal, + * provider-agnostic vision handoff. + * + * Two capabilities: + * 1. **prepareForProvider** (`prepareForProvider`): when a user message carries + * images but the active model cannot see them, this replaces each image chunk + * with a numbered placeholder (telling the model to call `consult_vision`) + * and registers the image data in a per-conversation registry for tool + * access. Vision-capable models pass through unchanged (images flow natively). + * 2. **consult_vision tool** (`consultVision`): opens a NEW conversation tab with + * a vision-capable model (resolved from the catalog — any provider), attaches + * the image(s) + the model's specific question, waits for the response, and + * returns the conversation ID + the vision model's answer. The model (e.g. + * GLM 5.2) directs the analysis — asking exactly what it needs — instead of + * receiving a pre-emptive generic dump. Follow-up questions go through the + * dispatch CLI (the conversation ID is the bridge), not another tool call. + * + * Effects (credential store, orchestrator, filesystem) are injected. The pure + * decisions live in `pure.ts`. This shell wires them. + */ + +import type { CredentialStore } from "@dispatch/credential-store"; +import type { + AgentEvent, + ChatMessage, + Chunk, + ImageInput, + Logger, + ModelInfo, + ProviderContract, +} from "@dispatch/kernel"; +import { defineService, type ServiceHandle } from "@dispatch/kernel"; +import { + collectTextFromStream, + findVisionModelName, + formatConsultationTitle, + formatConsultResult, + formatImagePlaceholder, + formatNoVisionPlaceholder, + isVisionCapable, +} from "./pure.js"; + +/** + * Minimal orchestrator interface the service needs to start vision consultation + * turns. Defined locally (not imported from session-orchestrator) to avoid a + * compile-time dependency — resolved lazily at runtime via a local handle keyed + * to the same service ID. + */ +export interface OrchestratorForVision { + readonly handleMessage: (input: { + readonly conversationId: string; + readonly text: string; + readonly onEvent: (event: AgentEvent) => void; + readonly modelName?: string; + readonly cwd?: string; + readonly images?: readonly ImageInput[]; + readonly systemPrompt?: string; + }) => Promise<void>; +} + +/** Local handle for the session-orchestrator service (same ID, no import dep). */ +export const orchestratorLocalHandle: ServiceHandle<OrchestratorForVision> = + defineService<OrchestratorForVision>("session-orchestrator/orchestrator"); + +/** + * Resolved vision model — a provider + its model id, ready to stream from. + */ +export interface ResolvedVisionModel { + readonly provider: ProviderContract; + readonly model: string; + readonly modelName: string; +} + +/** A registered image (looked up by the consult_vision tool via imageId). */ +interface RegisteredImage { + readonly url: string; + readonly mimeType?: string; +} + +/** + * Dependencies the service needs — all injected (no ambient state). + */ +export interface VisionHandoffDeps { + readonly credentialStore: CredentialStore; + /** Resolve a `<credentialName>/<model>` → its provider + model id. */ + readonly resolveModel: ( + modelName: string, + ) => { provider: ProviderContract; model: string } | undefined; + /** + * Read a file from disk as a base64 data URL. Injected so the shell controls + * the filesystem edge. Returns the data URL, or throws on error. + */ + readonly readFileAsDataUrl: (path: string, cwd?: string) => Promise<string>; + /** + * Lazily resolve the session-orchestrator (for starting vision consultation + * turns). Returns `undefined` when not available — `consult_vision` degrades + * with an error. Lazy so activation order doesn't matter. + */ + readonly resolveOrchestrator?: () => OrchestratorForVision | undefined; + /** + * Get the per-conversation cached image transcriptions (imageUrl → text). + * Used to avoid re-transcribing old images that were compacted to text on a + * previous turn. Optional — when absent, compaction still works but + * re-transcribes every turn (no caching). + */ + readonly getImageTranscriptions?: ( + conversationId: string, + ) => Promise<ReadonlyMap<string, string>>; + /** + * Upsert a single image transcription into the per-conversation cache. + * Optional — paired with getImageTranscriptions. + */ + readonly setImageTranscription?: ( + conversationId: string, + imageUrl: string, + transcription: string, + ) => Promise<void>; + /** + * Save an image data URL to a tmp file and return a compact URL + * (`/images/<conversationId>/<imageId>.<ext>`) that can be persisted in the + * conversation store instead of the full data URL (which would be megabytes). + * The frontend serves the image via `GET /images/...`; the provider resolves + * it back to a data URL via {@link resolveImageUrl} at runtime. When `undefined`, + * data URLs pass through unchanged (images persist in SQLite — the large-DB + * path, for environments without tmp file support). + */ + readonly saveImageToTmp?: ( + conversationId: string, + dataUrl: string, + mimeType?: string, + ) => Promise<string>; + /** + * Resolve a compact URL (`/images/...`) back to a data URL by reading the tmp + * file. Data URLs and HTTP URLs pass through unchanged. Paired with + * {@link saveImageToTmp}. + */ + readonly resolveImageUrl?: (url: string) => Promise<string>; + /** + * Delete a tmp image file (after it has been compacted to text — the + * transcription is cached, the raw image is no longer needed). Best-effort: + * errors are logged, not thrown. + */ + readonly deleteTmpImage?: (compactUrl: string) => Promise<void>; + /** + * Delete all tmp images for a conversation (on conversation close). + * Best-effort. + */ + readonly deleteConversationImages?: (conversationId: string) => Promise<void>; + /** + * Set the human-readable title of a conversation. Used to label vision + * consultation tabs with an `"IMAGE - "` prefix so they're visually + * distinguishable from normal conversation tabs. Backed by the conversation + * store's `setConversationTitle`. Optional — when absent, consultation tabs + * keep their default (question-derived) title. + */ + readonly setConversationTitle?: (conversationId: string, title: string) => Promise<void>; + /** Generate a new conversation ID for a consultation. Defaults to crypto.randomUUID. */ + readonly generateId?: () => string; + readonly logger?: Logger; +} + +export interface VisionHandoffService { + /** + * Whether a given model (by catalog name) is vision-capable. Uses the + * credential store's ModelInfo + the name heuristic. + */ + readonly isVisionCapable: (modelName: string | undefined) => Promise<boolean>; + + /** + * Store images to tmp files and return compact URLs. Each input image's data + * URL is saved to `/tmp/dispatch/images/<conversationId>/<uuid>.<ext>` and + * replaced with a compact HTTP path (`/images/<conversationId>/<uuid>.<ext>`) + * so the persisted conversation store holds a tiny string, not megabytes of + * base64. When `saveImageToTmp` is not configured, data URLs pass through + * unchanged (backward compatible). + */ + readonly storeImages: ( + conversationId: string, + images: readonly ImageInput[], + ) => Promise<readonly ImageInput[]>; + + /** + * Delete all tmp images for a conversation (on close). Best-effort. + */ + readonly purgeConversationImages: (conversationId: string) => Promise<void>; + + /** + * Resolve a vision-capable model from the catalog (any provider). Returns + * `undefined` when none is available. + */ + readonly resolveVisionModel: (excludeName?: string) => Promise<ResolvedVisionModel | undefined>; + + /** + * Transform a message list for the provider: if the active model is + * vision-capable, return messages unchanged (images pass through natively). + * If NOT vision-capable, replace every `image` chunk with a numbered + * placeholder (telling the model to call `consult_vision`) and register the + * image data in the per-conversation registry for tool access. The PERSISTED + * history is NOT modified — only what the provider sees. Never throws. + */ + readonly prepareForProvider: ( + messages: readonly ChatMessage[], + currentModelName: string | undefined, + opts?: { + readonly conversationId?: string; + readonly imageLimit?: number; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, + ) => Promise<readonly ChatMessage[]>; + + /** + * Look up a registered image by conversation ID + image ID. Returns + * `undefined` when the image isn't registered (e.g. after a server restart). + */ + readonly getRegisteredImage: ( + conversationId: string, + imageId: number, + ) => RegisteredImage | undefined; + + /** + * Open a NEW vision consultation conversation: attach image(s) + the model's + * question to a vision-capable model, wait for the response, and return the + * conversation ID + the vision model's answer. The model drives the analysis + * — it asks exactly what it needs. Follow-ups go through the dispatch CLI. + * + * @returns The conversation ID + the vision model's response text, or an + * error string (never throws — the tool surfaces it). + */ + readonly consultVision: ( + question: string, + opts: { + readonly conversationId: string; + readonly imageIds?: readonly number[]; + readonly path?: string; + readonly cwd?: string; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, + ) => Promise< + { readonly conversationId: string; readonly response: string } | { readonly error: string } + >; +} + +export const visionHandoffHandle: ServiceHandle<VisionHandoffService> = + defineService<VisionHandoffService>("vision-handoff/service"); + +/** Whether a message list contains any image chunks. Pure. */ +function hasImageChunks(messages: readonly ChatMessage[]): boolean { + return messages.some((m) => m.chunks.some((c) => c.type === "image")); +} + +export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHandoffService { + const log = deps.logger; + const generateId = deps.generateId ?? (() => crypto.randomUUID()); + + // Per-conversation image registry: conversationId → (imageId → image data). + // Populated by prepareForProvider; consulted by the consult_vision tool. + // In-memory only (cleared on restart — the user re-pastes if needed). + const imageRegistry = new Map<string, Map<number, RegisteredImage>>(); + + async function getInfo(modelName: string): Promise<ModelInfo | undefined> { + return deps.credentialStore.getModelInfo(modelName); + } + + async function resolveVisionModel( + excludeName?: string, + ): Promise<ResolvedVisionModel | undefined> { + const catalog = await deps.credentialStore.listCatalog(); + const name = await findVisionModelName(catalog, getInfo, excludeName); + if (name === undefined) return undefined; + const resolved = deps.resolveModel(name); + if (resolved === undefined) return undefined; + return { provider: resolved.provider, model: resolved.model, modelName: name }; + } + + /** + * Compact images for a vision-capable model: when the conversation has more + * image chunks than the limit, the oldest images are transcribed to text + * (one-time, cached in the conversation store) and stripped from the + * provider messages. Recent images (within the limit) stay native. + * + * The persisted history is NOT modified — only the provider's view. + * Transcriptions are cached so they're reused on subsequent turns (no + * re-transcription). When no caching deps are available, it still works but + * re-transcribes every turn. + */ + async function compactImagesForVisionModel( + messages: readonly ChatMessage[], + opts: + | { + readonly conversationId?: string; + readonly imageLimit?: number; + readonly signal?: AbortSignal; + readonly logger?: Logger; + } + | undefined, + currentModelName: string | undefined, + ): Promise<readonly ChatMessage[]> { + void currentModelName; // reserved for future model-specific compaction logic + const limit = opts?.imageLimit; + // No limit or limit <= 0 → pass all images through (compaction disabled). + if (limit === undefined || limit <= 0) return messages; + + // Collect all image chunks in order (oldest first, across all messages). + const imageEntries: { msgIdx: number; chunkIdx: number; url: string }[] = []; + for (const [mi, msg] of messages.entries()) { + for (const [ci, chunk] of msg.chunks.entries()) { + if (chunk.type === "image") { + imageEntries.push({ msgIdx: mi, chunkIdx: ci, url: chunk.url }); + } + } + } + + // If within the limit, pass everything through natively. + if (imageEntries.length <= limit) return messages; + + // The oldest (imageEntries.length - limit) images need transcription. + const toTranscribeCount = imageEntries.length - limit; + const toTranscribe = imageEntries.slice(0, toTranscribeCount); + + // Load cached transcriptions. + const convId = opts?.conversationId; + const cache = + convId !== undefined && deps.getImageTranscriptions !== undefined + ? await deps.getImageTranscriptions(convId) + : new Map<string, string>(); + + // Transcribe any that aren't cached yet (via the vision model). + const transcriptions = new Map<string, string>(cache); + const vision = await resolveVisionModel(); + for (const entry of toTranscribe) { + if (transcriptions.has(entry.url)) continue; + if (vision === undefined) { + // No vision model available for transcription — use a placeholder. + transcriptions.set( + entry.url, + "[Image was compacted — no vision model available to transcribe it.]", + ); + continue; + } + try { + const prompt = + "Describe this image in detail. Include visible text (transcribe verbatim), " + + "key objects, layout, and notable details. This description will replace " + + "the image in a conversation history, so be thorough."; + const userMessage: ChatMessage = { + role: "user", + chunks: [ + { type: "text", text: prompt }, + { type: "image", url: entry.url }, + ], + }; + const stream = vision.provider.stream([userMessage], [], { + model: vision.model, + systemPrompt: + "You are a vision assistant. Describe images faithfully and thoroughly. " + + "Do not use any tools — just use your vision to see the image and describe it directly.", + }); + const description = (await collectTextFromStream(stream)).trim(); + const text = + description.length > 0 ? description : "[Image transcription produced no output.]"; + transcriptions.set(entry.url, text); + // Cache it in the conversation store (if available). + if (convId !== undefined && deps.setImageTranscription !== undefined) { + await deps.setImageTranscription(convId, entry.url, text); + } + // The image has been transcribed to text — delete the tmp file + // (the transcription is cached, the raw image is no longer needed). + if (deps.deleteTmpImage !== undefined) { + try { + await deps.deleteTmpImage(entry.url); + } catch { + // Best-effort — don't let cleanup failure break the turn. + } + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + log?.warn("vision-handoff: image compaction transcription failed", { error: msg }); + transcriptions.set(entry.url, `[Image transcription failed: ${msg}]`); + } + } + + // Build the provider messages: replace transcribed images with text, + // keep recent images (within the limit) native. + const transcribedUrls = new Set(toTranscribe.map((e) => e.url)); + const result: ChatMessage[] = []; + for (const msg of messages) { + if (!msg.chunks.some((c) => c.type === "image")) { + result.push(msg); + continue; + } + const newChunks: Chunk[] = []; + for (const chunk of msg.chunks) { + if (chunk.type === "image" && transcribedUrls.has(chunk.url)) { + const transcription = transcriptions.get(chunk.url); + if (transcription !== undefined) { + newChunks.push({ type: "text", text: `[Compacted image]: ${transcription}` }); + } else { + newChunks.push(chunk); // fallback: keep the image + } + } else { + newChunks.push(chunk); + } + } + result.push({ role: msg.role, chunks: newChunks }); + } + return result; + } + + async function resolveImageUrlsInMessages( + messages: readonly ChatMessage[], + ): Promise<readonly ChatMessage[]> { + if (deps.resolveImageUrl === undefined) return messages; + let hasCompact = false; + for (const msg of messages) { + if (msg.chunks.some((c) => c.type === "image")) { + hasCompact = true; + break; + } + } + if (!hasCompact) return messages; + const result: ChatMessage[] = []; + for (const msg of messages) { + if (!msg.chunks.some((c) => c.type === "image")) { + result.push(msg); + continue; + } + const newChunks: Chunk[] = []; + for (const chunk of msg.chunks) { + if (chunk.type === "image") { + const dataUrl = await deps.resolveImageUrl!(chunk.url); + newChunks.push({ + type: "image", + url: dataUrl, + ...(chunk.mimeType !== undefined ? { mimeType: chunk.mimeType } : {}), + }); + } else { + newChunks.push(chunk); + } + } + result.push({ role: msg.role, chunks: newChunks }); + } + return result; + } + + const service: VisionHandoffService = { + async isVisionCapable(modelName: string | undefined): Promise<boolean> { + if (modelName === undefined) return false; + const info = await getInfo(modelName); + return isVisionCapable(modelName, info); + }, + + async storeImages( + conversationId: string, + images: readonly ImageInput[], + ): Promise<readonly ImageInput[]> { + if (deps.saveImageToTmp === undefined) return images; + const result: ImageInput[] = []; + for (const img of images) { + if (img.url.startsWith("data:")) { + const compactUrl = await deps.saveImageToTmp(conversationId, img.url, img.mimeType); + result.push({ + url: compactUrl, + ...(img.mimeType !== undefined ? { mimeType: img.mimeType } : {}), + }); + } else { + result.push(img); + } + } + return result; + }, + + async purgeConversationImages(conversationId: string): Promise<void> { + if (deps.deleteConversationImages === undefined) return; + try { + await deps.deleteConversationImages(conversationId); + } catch (err) { + log?.warn("vision-handoff: failed to purge conversation images", { + conversationId, + error: err instanceof Error ? err.message : String(err), + }); + } + }, + + resolveVisionModel, + + async prepareForProvider( + messages: readonly ChatMessage[], + currentModelName: string | undefined, + opts?: { + readonly conversationId?: string; + readonly imageLimit?: number; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, + ): Promise<readonly ChatMessage[]> { + // Fast path: no images anywhere → nothing to do. + if (!hasImageChunks(messages)) return messages; + + // Resolve compact URLs (/images/...) → data URLs for the provider. + // The persisted chunks store compact URLs (tiny strings); the provider + // needs data URLs (read from tmp files at runtime). + const resolved = await resolveImageUrlsInMessages(messages); + + const isCapable = + currentModelName !== undefined && + (await isVisionCapable(currentModelName, await getInfo(currentModelName))); + + // ── Vision-capable model: image compaction ────────────────────────── + // When the conversation has more images than the limit, the oldest images + // are transcribed to text (one-time, cached) and stripped from the + // provider messages. Recent images (within the limit) stay native. + if (isCapable) { + return compactImagesForVisionModel(resolved, opts, currentModelName); + } + + // ── Non-vision model: placeholders + consult_vision ────────────────── + const vision = await resolveVisionModel(); + const convId = opts?.conversationId; + + const placeholderFn = + vision !== undefined && convId !== undefined + ? (id: number) => formatImagePlaceholder(id) + : () => formatNoVisionPlaceholder(); + + // Replace each image chunk with a numbered placeholder. Assign sequential + // 1-based IDs across all messages and register each image in the + // per-conversation registry so the consult_vision tool can look it up. + let seqId = 0; + const result: ChatMessage[] = []; + for (const msg of resolved) { + if (!msg.chunks.some((c) => c.type === "image")) { + result.push(msg); + continue; + } + const newChunks: Chunk[] = []; + for (const chunk of msg.chunks) { + if (chunk.type === "image") { + seqId++; + if (convId !== undefined && vision !== undefined) { + let convImages = imageRegistry.get(convId); + if (convImages === undefined) { + convImages = new Map(); + imageRegistry.set(convId, convImages); + } + convImages.set(seqId, { + url: chunk.url, + ...(chunk.mimeType !== undefined ? { mimeType: chunk.mimeType } : {}), + }); + } + newChunks.push({ type: "text", text: placeholderFn(seqId) }); + } else { + newChunks.push(chunk); + } + } + result.push({ role: msg.role, chunks: newChunks }); + } + return result; + }, + + getRegisteredImage(conversationId: string, imageId: number): RegisteredImage | undefined { + return imageRegistry.get(conversationId)?.get(imageId); + }, + + async consultVision( + question: string, + opts: { + readonly conversationId: string; + readonly imageIds?: readonly number[]; + readonly path?: string; + readonly cwd?: string; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, + ): Promise< + { readonly conversationId: string; readonly response: string } | { readonly error: string } + > { + const orchestrator = deps.resolveOrchestrator?.(); + if (orchestrator === undefined) { + return { + error: "The session orchestrator is not available — cannot start a vision consultation.", + }; + } + + const vision = await resolveVisionModel(); + if (vision === undefined) { + return { + error: + "No vision-capable model is available in the catalog. Install or configure one (e.g. kimi) to enable image analysis.", + }; + } + + // Collect image data URLs to attach. + const images: ImageInput[] = []; + if (opts.imageIds !== undefined) { + for (const id of opts.imageIds) { + const img = service.getRegisteredImage(opts.conversationId, id); + if (img === undefined) { + return { + error: `Image ${id} is not registered. It may have been lost after a server restart — ask the user to re-paste the image.`, + }; + } + images.push({ + url: img.url, + ...(img.mimeType !== undefined ? { mimeType: img.mimeType } : {}), + }); + } + } + if (opts.path !== undefined) { + try { + const dataUrl = await deps.readFileAsDataUrl(opts.path, opts.cwd); + images.push({ url: dataUrl }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { error: `Failed to read image file "${opts.path}": ${msg}` }; + } + } + if (images.length === 0) { + return { + error: + "No image to consult about. Provide imageIds (for pasted images) or path (for a file).", + }; + } + + // Start a NEW conversation with the vision model. + const consultationId = generateId(); + log?.info("vision-handoff: starting consultation", { + consultationId, + visionModel: vision.modelName, + imageCount: images.length, + fromConversation: opts.conversationId, + }); + + // Label the consultation tab with an "IMAGE - " prefix so it's visually + // distinguishable from normal conversation tabs. Set BEFORE the turn + // starts so the tab shows the correct title from the first moment (the + // store keeps a non-"Untitled" title on first message append). + if (deps.setConversationTitle !== undefined) { + try { + await deps.setConversationTitle(consultationId, formatConsultationTitle(question)); + } catch (err) { + // Best-effort — don't let a title-write failure break the consultation. + log?.warn("vision-handoff: failed to set consultation title", { + consultationId, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + let responseText = ""; + let errorMessage = ""; + try { + await orchestrator.handleMessage({ + conversationId: consultationId, + text: question, + images, + modelName: vision.modelName, + ...(opts.cwd !== undefined ? { cwd: opts.cwd } : {}), + systemPrompt: + "You are a vision assistant. A developer who cannot see images is asking you specific " + + "questions about an image they attached. Answer their question precisely and thoroughly. " + + "Do not use any tools unless specifically asked to — just use your vision to see the " + + "image and describe it directly.", + onEvent: (event: AgentEvent) => { + if (event.type === "text-delta") { + responseText += event.delta; + } else if (event.type === "error") { + errorMessage = event.message; + } + }, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { error: `Vision consultation failed: ${msg}` }; + } + + if (errorMessage.length > 0 && responseText.trim().length === 0) { + return { error: `Vision consultation failed: ${errorMessage}` }; + } + + const response = formatConsultResult(consultationId, responseText); + return { conversationId: consultationId, response }; + }, + }; + + return service; +} diff --git a/packages/vision-handoff/src/tool.ts b/packages/vision-handoff/src/tool.ts new file mode 100644 index 0000000..86be2ed --- /dev/null +++ b/packages/vision-handoff/src/tool.ts @@ -0,0 +1,137 @@ +/** + * consult_vision tool — lets any model (vision-capable or not) consult a + * vision-capable model about an image by opening a NEW conversation tab. + * + * The tool attaches image(s) + the model's specific question to a vision-capable + * model (resolved from the catalog — e.g. Kimi), waits for the response, and + * returns the conversation ID + the vision model's answer. The MODEL directs the + * analysis — it asks exactly what it needs to know — instead of receiving a + * pre-emptive generic dump. + * + * For images PASTED into the chat, the model references them by `imageIds` (from + * the "[Image N attached]" placeholders the orchestrator injected). For image + * FILES on disk, the model passes a `path`. + * + * Follow-up questions are NOT handled by this tool — the model uses the dispatch + * CLI to continue the vision conversation (the returned conversation ID is the + * bridge; the model can load the `dispatch-cli` skill for the exact commands). + */ + +import type { ToolContract, ToolExecuteContext, ToolResult } from "@dispatch/kernel"; +import type { VisionHandoffService } from "./service.js"; + +export function createConsultVisionTool(service: VisionHandoffService): ToolContract { + return { + name: "consult_vision", + description: + "Consult a vision-capable model (e.g. Kimi) about an image by opening a new " + + "conversation tab. Attaches the image(s) + your specific question, waits for " + + "the vision model's response, and returns the conversation ID + the answer. " + + "Use this when you cannot view an image (e.g. a pasted screenshot or diagram) " + + "and need to know what it shows — ask a SPECIFIC question (e.g. 'What error " + + "message is on line 12?' rather than 'describe this image'). The conversation " + + "ID is returned so follow-up questions can be asked via the dispatch CLI.", + parameters: { + type: "object", + properties: { + question: { + type: "string", + description: + "Your specific question about the image. Be precise — the vision model " + + "will answer exactly this. E.g. 'What error message is displayed?' or " + + "'Compare the layout of these two screenshots.'", + }, + imageIds: { + type: "array", + items: { type: "number" }, + description: + "The IDs of pasted images to attach (from the '[Image N attached]' " + + "placeholders in the conversation). Pass multiple to attach several " + + "images to one consultation (e.g. [1, 2] to compare them).", + }, + path: { + type: "string", + description: + "Path to an image FILE on disk to attach (alternative to imageIds for " + + "code-referenced images). Relative paths resolve against the cwd.", + }, + }, + required: ["question"], + }, + concurrencySafe: true, + async execute(args: unknown, ctx: ToolExecuteContext): Promise<ToolResult> { + const input = args as { + question?: unknown; + imageIds?: unknown; + path?: unknown; + } | null; + + const question = input?.question; + if (typeof question !== "string" || question.trim().length === 0) { + return { + content: "Error: 'question' is required and must be a non-empty string.", + isError: true, + }; + } + + const imageIds = input?.imageIds; + const path = input?.path; + + // Parse imageIds (must be an array of numbers if present). + let parsedImageIds: number[] | undefined; + if (imageIds !== undefined) { + if (!Array.isArray(imageIds)) { + return { content: "Error: 'imageIds' must be an array of numbers.", isError: true }; + } + parsedImageIds = imageIds.filter((n): n is number => typeof n === "number"); + if (parsedImageIds.length === 0) { + return { content: "Error: 'imageIds' must contain at least one number.", isError: true }; + } + } + + // path must be a string if present. + let parsedPath: string | undefined; + if (path !== undefined) { + if (typeof path !== "string" || path.trim().length === 0) { + return { content: "Error: 'path' must be a non-empty string.", isError: true }; + } + parsedPath = path; + } + + // At least one image source is required. + if (parsedImageIds === undefined && parsedPath === undefined) { + return { + content: + "Error: provide 'imageIds' (for pasted images) or 'path' (for a file) " + + "to attach an image to the consultation.", + isError: true, + }; + } + + const span = ctx.log.span("consult_vision.execute", { + imageCount: (parsedImageIds?.length ?? 0) + (parsedPath !== undefined ? 1 : 0), + }); + try { + const result = await service.consultVision(question, { + conversationId: ctx.conversationId ?? "", + ...(parsedImageIds !== undefined ? { imageIds: parsedImageIds } : {}), + ...(parsedPath !== undefined ? { path: parsedPath } : {}), + ...(ctx.cwd !== undefined ? { cwd: ctx.cwd } : {}), + signal: ctx.signal, + logger: ctx.log, + }); + span.end({ attrs: { ok: !("error" in result) } }); + if ("error" in result) { + return { content: result.error, isError: true }; + } + return { content: result.response }; + } catch (err: unknown) { + span.end({ err }); + return { + content: `Error during vision consultation: ${err instanceof Error ? err.message : String(err)}`, + isError: true, + }; + } + }, + }; +} |