diff options
| author | Adam Malczewski <[email protected]> | 2026-06-27 18:12:04 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-27 18:12:04 +0900 |
| commit | e76790fef11de3cc33f60689f9301030ed740cd6 (patch) | |
| tree | 42032276165ec91ec9eebb27738165296729953f | |
| parent | 72d08ddffbbf70d73db8d223aac20937f662560f (diff) | |
| download | dispatch-e76790fef11de3cc33f60689f9301030ed740cd6.tar.gz dispatch-e76790fef11de3cc33f60689f9301030ed740cd6.zip | |
feat(vision-handoff): model-directed consult_vision tool replacing auto-transcription
| -rw-r--r-- | packages/session-orchestrator/src/orchestrator.ts | 17 | ||||
| -rw-r--r-- | packages/vision-handoff/src/extension.ts | 67 | ||||
| -rw-r--r-- | packages/vision-handoff/src/index.ts | 8 | ||||
| -rw-r--r-- | packages/vision-handoff/src/pure.test.ts | 51 | ||||
| -rw-r--r-- | packages/vision-handoff/src/pure.ts | 85 | ||||
| -rw-r--r-- | packages/vision-handoff/src/service.test.ts | 252 | ||||
| -rw-r--r-- | packages/vision-handoff/src/service.ts | 371 | ||||
| -rw-r--r-- | packages/vision-handoff/src/tool.ts | 127 |
8 files changed, 623 insertions, 355 deletions
diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts index ac1eaf4..4f4bb3e 100644 --- a/packages/session-orchestrator/src/orchestrator.ts +++ b/packages/session-orchestrator/src/orchestrator.ts @@ -43,16 +43,20 @@ import type { ToolAssembly } from "./tools-filter.js"; * off cleanly when the extension isn't loaded (images pass through unchanged, * which is correct for vision-capable models and a no-op for text-only turns). * - * `transcribeForProvider` transforms a message list for the provider: if the + * `prepareForProvider` transforms a message list for the provider: if the * active model is vision-capable, messages pass through unchanged; otherwise - * image chunks are replaced with text descriptions (transcribed via a - * vision-capable model). Never throws — degrades to placeholders. + * image chunks are replaced with numbered placeholders (telling the model to + * call `consult_vision`) and the images are registered for tool access. */ export interface VisionHandoffService { - readonly transcribeForProvider: ( + readonly prepareForProvider: ( messages: readonly ChatMessage[], currentModelName: string | undefined, - opts?: { readonly signal?: AbortSignal; readonly logger?: Logger }, + opts?: { + readonly conversationId?: string; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, ) => Promise<readonly ChatMessage[]>; } @@ -772,10 +776,11 @@ export function createSessionOrchestrator( const visionHandoff = deps.resolveVisionHandoff?.(); let providerMessages: readonly ChatMessage[] = [...history, userMsg]; if (visionHandoff !== undefined) { - providerMessages = await visionHandoff.transcribeForProvider( + providerMessages = await visionHandoff.prepareForProvider( providerMessages, effectiveModelName, { + conversationId, signal: controller.signal, ...(turnLogger !== undefined ? { logger: turnLogger } : {}), }, diff --git a/packages/vision-handoff/src/extension.ts b/packages/vision-handoff/src/extension.ts index aa745b7..2f75a6b 100644 --- a/packages/vision-handoff/src/extension.ts +++ b/packages/vision-handoff/src/extension.ts @@ -1,15 +1,17 @@ /** * vision-handoff extension — registers the universal vision handoff service + - * the `read_image` tool. + * the `consult_vision` tool. * - * The service performs provider-agnostic vision handoff: it resolves a - * vision-capable model from the catalog (any provider), streams an image to it - * via the standard `ProviderContract.stream` interface, and folds the textual - * description back — so a non-vision model (e.g. glm-5.2) can still reason about - * images, and any model can analyze image FILES referenced in code. + * The service performs provider-agnostic vision handoff: when a non-vision model + * (e.g. glm-5.2) receives an image, it replaces the image with a numbered + * placeholder and registers it for tool access. The `consult_vision` tool opens + * a NEW conversation tab with a vision-capable model (e.g. Kimi), attaches the + * image + the model's specific question, and returns the conversation ID + the + * vision model's answer. Follow-ups go through the dispatch CLI. * - * Effects (filesystem, fetch) live here in the shell, injected into the service. - * The pure decisions live in `pure.ts`. No `console.*`; logging via `host.logger`. + * Effects (filesystem, orchestrator) live here in the shell, injected into the + * service. The pure decisions live in `pure.ts`. No `console.*`; logging via + * `host.logger`. */ import { readFile } from "node:fs/promises"; @@ -17,8 +19,12 @@ import { extname, isAbsolute, resolve as pathResolve } from "node:path"; import type { CredentialStore } from "@dispatch/credential-store"; import { credentialStoreHandle } from "@dispatch/credential-store"; import type { Extension, HostAPI, Manifest } from "@dispatch/kernel"; -import { createVisionHandoffService, visionHandoffHandle } from "./service.js"; -import { createReadImageTool } from "./tool.js"; +import { + createVisionHandoffService, + orchestratorLocalHandle, + visionHandoffHandle, +} from "./service.js"; +import { createConsultVisionTool } from "./tool.js"; export const manifest: Manifest = { id: "vision-handoff", @@ -28,7 +34,7 @@ export const manifest: Manifest = { trust: "bundled", activation: "eager", capabilities: { network: true }, - contributes: { services: ["vision-handoff/service"], tools: ["read_image"] }, + contributes: { services: ["vision-handoff/service"], tools: ["consult_vision"] }, }; /** MIME types for recognized image extensions. */ @@ -54,30 +60,11 @@ async function readFileAsDataUrl(path: string, cwd?: string): Promise<string> { return `data:${mime};base64,${buf.toString("base64")}`; } -/** - * Fetch an HTTP(S) image URL and convert it to a base64 data URL (so it can be - * sent to the vision model inline, regardless of whether the provider can fetch - * remote URLs). The shell edge — real `globalThis.fetch`. - */ -async function fetchUrlAsDataUrl(url: string): Promise<string> { - const res = await fetch(url); - if (!res.ok) { - throw new Error(`Failed to fetch image: HTTP ${res.status}`); - } - const buf = new Uint8Array(await res.arrayBuffer()); - const mime = res.headers.get("content-type") ?? "image/png"; - // Buffer/base64 in Bun + Node. Convert byte-by-byte without non-null asserts. - let binary = ""; - for (const byte of buf) binary += String.fromCharCode(byte); - const base64 = btoa(binary); - return `data:${mime};base64,${base64}`; -} - export async function activate(host: HostAPI): Promise<void> { const credentialStore = host.getService(credentialStoreHandle) as CredentialStore | undefined; if (credentialStore === undefined) { host.logger.warn( - "vision-handoff: credential-store service not available. The read_image tool and image transcription are disabled.", + "vision-handoff: credential-store service not available. The consult_vision tool and image handoff are disabled.", ); return; } @@ -94,13 +81,25 @@ export async function activate(host: HostAPI): Promise<void> { credentialStore, resolveModel, readFileAsDataUrl, - fetchUrlAsDataUrl, + // Lazily resolve the session-orchestrator (for starting vision consultation + // turns). By the time consult_vision is called at runtime, all extensions + // have activated. The activated-manifests guard avoids a getService throw + // when the orchestrator isn't loaded. + resolveOrchestrator: () => { + const loaded = host.getExtensions().some((m) => m.id === "session-orchestrator"); + if (!loaded) return undefined; + try { + return host.getService(orchestratorLocalHandle); + } catch { + return undefined; + } + }, logger: host.logger.child({ extensionId: "vision-handoff" }), }); host.provideService(visionHandoffHandle, service); - host.defineTool(createReadImageTool(service)); - host.logger.info("vision-handoff: registered (read_image tool + transcription service)"); + host.defineTool(createConsultVisionTool(service)); + host.logger.info("vision-handoff: registered (consult_vision tool + handoff service)"); } export const extension: Extension = { manifest, activate }; diff --git a/packages/vision-handoff/src/index.ts b/packages/vision-handoff/src/index.ts index 4a13e65..2713346 100644 --- a/packages/vision-handoff/src/index.ts +++ b/packages/vision-handoff/src/index.ts @@ -1,19 +1,21 @@ export { extension, manifest } from "./extension.js"; export { - buildTranscriptionPrompt, collectTextFromStream, findVisionModelName, + formatConsultResult, + formatImagePlaceholder, formatNoVisionPlaceholder, - formatTranscriptionText, isVisionCapable, } from "./pure.js"; export type { + OrchestratorForVision, ResolvedVisionModel, VisionHandoffDeps, VisionHandoffService, } from "./service.js"; export { createVisionHandoffService, + orchestratorLocalHandle, visionHandoffHandle, } from "./service.js"; -export { createReadImageTool } from "./tool.js"; +export { createConsultVisionTool } from "./tool.js"; diff --git a/packages/vision-handoff/src/pure.test.ts b/packages/vision-handoff/src/pure.test.ts index 89dac72..4198d00 100644 --- a/packages/vision-handoff/src/pure.test.ts +++ b/packages/vision-handoff/src/pure.test.ts @@ -1,11 +1,11 @@ import type { ModelInfo, ProviderEvent } from "@dispatch/kernel"; import { describe, expect, it } from "vitest"; import { - buildTranscriptionPrompt, collectTextFromStream, findVisionModelName, + formatConsultResult, + formatImagePlaceholder, formatNoVisionPlaceholder, - formatTranscriptionText, isVisionCapable, } from "./pure.js"; @@ -43,7 +43,7 @@ describe("findVisionModelName", () => { return map[name]; }; - it("finds the first kimi-family model via name heuristic (no async lookup needed)", async () => { + it("finds the first kimi-family model via name heuristic", async () => { const name = await findVisionModelName( ["umans/glm-5.2", "umans/kimi-k2.7", "umans/llama-vision"], getInfo, @@ -90,7 +90,7 @@ describe("collectTextFromStream", () => { expect(text).toBe("Hello world!"); }); - it("ignores non-text events (reasoning, usage, tool-call, finish)", async () => { + it("ignores non-text events", async () => { const events: ProviderEvent[] = [ { type: "reasoning-delta", delta: "thinking..." }, { type: "text-delta", delta: "answer" }, @@ -115,27 +115,38 @@ describe("collectTextFromStream", () => { }); }); -describe("prompt + formatting helpers", () => { - it("buildTranscriptionPrompt includes focus when a question is given", () => { - const prompt = buildTranscriptionPrompt("What error is shown?"); - expect(prompt).toContain("Describe this image in detail"); - expect(prompt).toContain('The user asked: "What error is shown?"'); +describe("formatImagePlaceholder", () => { + it("includes the image ID and mentions consult_vision", () => { + const text = formatImagePlaceholder(1); + expect(text).toContain("Image 1"); + expect(text).toContain("consult_vision"); + expect(text).toContain("imageIds=[1]"); }); - it("buildTranscriptionPrompt omits focus when no question", () => { - const prompt = buildTranscriptionPrompt(undefined); - expect(prompt).toContain("Describe this image in detail"); - expect(prompt).not.toContain("The user asked"); - }); - - it("formatTranscriptionText names the vision model", () => { - expect(formatTranscriptionText("a red car", "umans/kimi-k2.7")).toBe( - "[Image analysis (via umans/kimi-k2.7)]: a red car", - ); + it("increments the ID for each image", () => { + expect(formatImagePlaceholder(2)).toContain("Image 2"); + expect(formatImagePlaceholder(2)).toContain("imageIds=[2]"); }); +}); - it("formatNoVisionPlaceholder explains the limitation", () => { +describe("formatNoVisionPlaceholder", () => { + it("explains the limitation", () => { const text = formatNoVisionPlaceholder(); expect(text).toContain("no vision-capable model"); }); }); + +describe("formatConsultResult", () => { + it("includes the conversation ID, the response, and the dispatch CLI hint", () => { + const result = formatConsultResult("abc-123", "The error is on line 12."); + expect(result).toContain("abc-123"); + expect(result).toContain("The error is on line 12."); + expect(result).toContain("dispatch CLI"); + }); + + it("trims the response", () => { + const result = formatConsultResult("c1", " spaced "); + expect(result).toContain("spaced"); + expect(result).not.toContain("spaced "); + }); +}); diff --git a/packages/vision-handoff/src/pure.ts b/packages/vision-handoff/src/pure.ts index 11eeefc..5eeb1a3 100644 --- a/packages/vision-handoff/src/pure.ts +++ b/packages/vision-handoff/src/pure.ts @@ -2,9 +2,10 @@ * Pure decision helpers for the vision handoff. * * No I/O, no ambient state. The shell (the extension + the service) injects the - * effects (credential store lookups, provider streaming). This module owns only - * the policy: which model is vision-capable, how to build a transcription - * request, and how to fold a provider's streamed text into a description. + * effects (credential store lookups, orchestrator, provider streaming). This + * module owns only the policy: which model is vision-capable, how to format + * image placeholders for non-vision models, and how to format the + * consultation tool's result. */ import type { ModelInfo, ProviderEvent } from "@dispatch/kernel"; @@ -36,9 +37,7 @@ export function isVisionCapable( /** * Find the first vision-capable model name in a catalog, given a lookup that * resolves a `<credentialName>/<model>` → `ModelInfo`. Returns `undefined` when - * no vision-capable model is available (the handoff degrades: images are - * replaced with a placeholder note). Pure given the (async) lookup — no - * ambient state, no side effects. + * no vision-capable model is available. Pure given the (async) lookup. * * @param catalog The full list of model names (`<credentialName>/<model>`). * @param getInfo Async lookup of a model name → ModelInfo (from the credential store). @@ -52,8 +51,7 @@ export async function findVisionModelName( for (const name of catalog) { if (exclude !== undefined && name === exclude) continue; // Fast path: the name heuristic lets us short-circuit without an async - // lookup for known vision families (kimi). This avoids a round-trip to - // listModels for the common case. + // lookup for known vision families (kimi). const slash = name.indexOf("/"); const modelId = slash >= 0 ? name.slice(slash + 1) : name; if (isVisionModelId(modelId)) return name; @@ -64,11 +62,10 @@ export async function findVisionModelName( } /** - * Fold a provider's streamed events into a single text string (the - * transcription). Pure given the async iterable — collects `text-delta` events, - * ignores everything else (reasoning, usage, tool-calls, errors). If the stream - * yields an error event, it is surfaced as a thrown Error so the caller can - * decide how to degrade (placeholder vs. fail). Pure: input → output, no I/O. + * Fold a provider's streamed events into a single text string. Pure given the + * async iterable — collects `text-delta` events, ignores everything else + * (reasoning, usage, tool-calls). If the stream yields an error event, it is + * surfaced as a thrown Error so the caller can decide how to degrade. */ export async function collectTextFromStream(stream: AsyncIterable<ProviderEvent>): Promise<string> { let text = ""; @@ -83,43 +80,26 @@ export async function collectTextFromStream(stream: AsyncIterable<ProviderEvent> } /** - * Build the prompt sent to the vision model to transcribe an image. Kept here - * (pure) so the prompt is testable and stable. The prompt asks for a thorough - * description so the text-only model has enough detail to reason about the - * image's contents. Pure. + * Format the placeholder text that replaces an `image` chunk when a non-vision + * model is active. The placeholder tells the model an image is attached and it + * should call `consult_vision` to analyze it — the model drives the analysis + * (asking a specific question) rather than receiving a pre-emptive generic dump. * - * @param userQuestion The user's own message text (may be empty) — passed so - * the vision model can tailor its description to what the user actually asked. + * @param imageId The 1-based ID assigned to this image (used by the tool to + * look up the registered image data). + * Pure. */ -export function buildTranscriptionPrompt(userQuestion: string | undefined): string { - const focus = - userQuestion && userQuestion.trim().length > 0 - ? `\n\nThe user asked: "${userQuestion.trim()}". Focus your description on what is relevant to that question, but still describe the whole image.` - : ""; +export function formatImagePlaceholder(imageId: number): string { return ( - "Describe this image in detail. Include: the overall scene/subject, " + - "visible text (transcribe verbatim), key objects, layout, colors, and any " + - "notable details a developer or user would need to understand the image." + - focus + `[Image ${imageId} attached — you cannot view images. Call the ` + + `consult_vision tool with imageIds=[${imageId}] and a specific question ` + + `to analyze it via a vision-capable model.]` ); } /** - * Format a single image's transcription as a text chunk string for the - * persisted user message. The note names the vision model so the consumer knows - * the description's provenance. Pure. - */ -export function formatTranscriptionText( - description: string, - visionModelName: string | undefined, -): string { - const source = visionModelName ?? "vision model"; - return `[Image analysis (via ${source})]: ${description}`; -} - -/** * Placeholder text used when NO vision-capable model is available (the - * degraded path). Pure. + * degraded path — the tool cannot function). Pure. */ export function formatNoVisionPlaceholder(): string { return ( @@ -127,3 +107,24 @@ export function formatNoVisionPlaceholder(): string { "Install or configure a vision-capable model (e.g. kimi) to enable image analysis.]" ); } + +/** + * Format the `consult_vision` tool's result string. Returns the conversation ID + * (so the model / user can continue the vision consultation), the vision model's + * response, and a note that follow-up questions use the dispatch CLI (the model + * can load the `dispatch-cli` skill for the exact commands). + * + * Pure. + * + * @param conversationId The new vision consultation conversation ID. + * @param response The vision model's answer to the model's question. + */ +export function formatConsultResult(conversationId: string, response: string): string { + const trimmed = response.trim(); + return ( + `Vision consultation opened in conversation ${conversationId}.\n\n` + + `Response: ${trimmed}\n\n` + + `To ask follow-up questions about this image, use the dispatch CLI ` + + `(conversation: ${conversationId}).` + ); +} diff --git a/packages/vision-handoff/src/service.test.ts b/packages/vision-handoff/src/service.test.ts index fe99d17..73c647b 100644 --- a/packages/vision-handoff/src/service.test.ts +++ b/packages/vision-handoff/src/service.test.ts @@ -1,9 +1,9 @@ import type { + AgentEvent, ChatMessage, ModelInfo, ProviderContract, ProviderEvent, - ProviderStreamOptions, ToolContract, } from "@dispatch/kernel"; import { describe, expect, it, vi } from "vitest"; @@ -21,7 +21,6 @@ function makeVisionProvider( ( messages: readonly ChatMessage[], _tools: readonly ToolContract[], - _opts?: ProviderStreamOptions, ): AsyncIterable<ProviderEvent> => { const img = messages.flatMap((m) => m.chunks).find((c) => c.type === "image"); const url = img && img.type === "image" ? img.url : ""; @@ -91,46 +90,11 @@ describe("VisionHandoffService.resolveVisionModel", () => { it("excludes the given model", async () => { const svc = createVisionHandoffService(makeDeps()); const vision = await svc.resolveVisionModel("umans/kimi-k2.7"); - // kimi is the only vision model; excluding it → undefined. expect(vision).toBeUndefined(); }); }); -describe("VisionHandoffService.transcribeImage", () => { - it("returns a formatted description from the vision model", async () => { - const svc = createVisionHandoffService(makeDeps()); - const result = await svc.transcribeImage("data:image/png;base64,xxx", "what is this?"); - expect(result).toBe( - "[Image analysis (via umans/kimi-k2.7)]: DESCRIPTION of data:image/png;base64,xxx", - ); - }); - - it("returns a placeholder when no vision model is available", async () => { - const deps = makeDeps(); - // Empty catalog → no vision model. - (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]); - const svc = createVisionHandoffService(deps); - const result = await svc.transcribeImage("data:image/png;base64,xxx", undefined); - expect(result).toContain("no vision-capable model"); - }); - - it("returns an error note when the vision stream errors", async () => { - const errorProvider: ProviderContract = { - id: "umans", - stream: vi.fn(async function* (): AsyncIterable<ProviderEvent> { - yield { type: "error", message: "vision API down" }; - }), - }; - const deps = makeDeps({ - resolveModel: vi.fn(() => ({ provider: errorProvider, model: "kimi-k2.7" })), - }); - const svc = createVisionHandoffService(deps); - const result = await svc.transcribeImage("data:image/png;base64,xxx", undefined); - expect(result).toContain("Image analysis failed: vision API down"); - }); -}); - -describe("VisionHandoffService.transcribeForProvider", () => { +describe("VisionHandoffService.prepareForProvider", () => { it("passes messages through unchanged when the model is vision-capable", async () => { const deps = makeDeps(); const svc = createVisionHandoffService(deps); @@ -143,19 +107,19 @@ describe("VisionHandoffService.transcribeForProvider", () => { ], }, ]; - const result = await svc.transcribeForProvider(messages, "umans/kimi-k2.7"); - expect(result).toBe(messages); // same reference — no copy, no transcription + const result = await svc.prepareForProvider(messages, "umans/kimi-k2.7"); + expect(result).toBe(messages); // same reference — no copy, no change }); it("passes messages through unchanged when there are no images", async () => { const deps = makeDeps(); const svc = createVisionHandoffService(deps); const messages: ChatMessage[] = [{ role: "user", chunks: [{ type: "text", text: "hi" }] }]; - const result = await svc.transcribeForProvider(messages, "umans/glm-5.2"); + const result = await svc.prepareForProvider(messages, "umans/glm-5.2"); expect(result).toBe(messages); }); - it("transcribes image chunks to text for a non-vision model", async () => { + it("replaces image chunks with numbered placeholders for a non-vision model", async () => { const deps = makeDeps(); const svc = createVisionHandoffService(deps); const messages: ChatMessage[] = [ @@ -167,76 +131,198 @@ describe("VisionHandoffService.transcribeForProvider", () => { ], }, ]; - const result = await svc.transcribeForProvider(messages, "umans/glm-5.2"); + const result = await svc.prepareForProvider(messages, "umans/glm-5.2", { + conversationId: "conv-1", + }); expect(result).toHaveLength(1); const chunks = result[0]?.chunks; expect(chunks).toHaveLength(2); + // Text chunk unchanged. expect(chunks?.[0]).toEqual({ type: "text", text: "Describe this" }); - // The image chunk was replaced with a transcribed text chunk. + // Image chunk → placeholder text. expect(chunks?.[1]?.type).toBe("text"); - expect((chunks?.[1] as { text: string }).text).toContain("Image analysis"); - expect((chunks?.[1] as { text: string }).text).toContain("img1"); + const placeholder = (chunks?.[1] as { text: string }).text; + expect(placeholder).toContain("Image 1"); + expect(placeholder).toContain("consult_vision"); }); - it("caches transcription per unique image URL within a call", async () => { + it("assigns sequential image IDs across multiple messages", async () => { const deps = makeDeps(); const svc = createVisionHandoffService(deps); const messages: ChatMessage[] = [ - { - role: "user", - chunks: [ - { type: "image", url: "data:image/png;base64,same" }, - { type: "image", url: "data:image/png;base64,same" }, - ], - }, + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,a" }] }, + { role: "assistant", chunks: [{ type: "text", text: "ok" }] }, + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,b" }] }, ]; - const result = await svc.transcribeForProvider(messages, "umans/glm-5.2"); - const chunks = result[0]?.chunks; - // Both image chunks → text, same description (cached). - expect(chunks).toHaveLength(2); - expect((chunks?.[0] as { text: string }).text).toBe((chunks?.[1] as { text: string }).text); - // The vision provider was called only once (cache hit on the second). - const provider = deps.resolveModel("umans/kimi-k2.7")?.provider; - expect((provider?.stream as ReturnType<typeof vi.fn>).mock.calls).toHaveLength(1); + const result = await svc.prepareForProvider(messages, "umans/glm-5.2", { + conversationId: "conv-1", + }); + // First image → Image 1, second → Image 2. + expect((result[0]?.chunks[0] as { text: string }).text).toContain("Image 1"); + // Assistant message unchanged. + expect(result[1]?.chunks[0]?.type).toBe("text"); + expect((result[2]?.chunks[0] as { text: string }).text).toContain("Image 2"); }); - it("transcribes images in history messages too (non-vision model)", async () => { + it("registers images so getRegisteredImage can look them up", async () => { const deps = makeDeps(); const svc = createVisionHandoffService(deps); const messages: ChatMessage[] = [ - { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,hist" }] }, - { role: "assistant", chunks: [{ type: "text", text: "got it" }] }, - { role: "user", chunks: [{ type: "text", text: "and now?" }] }, + { + role: "user", + chunks: [{ type: "image", url: "data:image/png;base64,registered" }], + }, ]; - const result = await svc.transcribeForProvider(messages, "umans/glm-5.2"); - // First message's image chunk is now text. - expect(result[0]?.chunks[0]?.type).toBe("text"); - expect((result[0]?.chunks[0] as { text: string }).text).toContain("Image analysis"); - // Assistant message unchanged. - expect(result[1]?.chunks[0]?.type).toBe("text"); - // Last user message unchanged. - expect(result[2]?.chunks[0]).toEqual({ type: "text", text: "and now?" }); + await svc.prepareForProvider(messages, "umans/glm-5.2", { conversationId: "conv-42" }); + const img = svc.getRegisteredImage("conv-42", 1); + expect(img?.url).toBe("data:image/png;base64,registered"); }); - it("uses a placeholder when no vision model is available (non-vision model)", async () => { + it("uses no-vision placeholder when no vision model is available", async () => { const deps = makeDeps(); (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]); const svc = createVisionHandoffService(deps); const messages: ChatMessage[] = [ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,abc" }] }, ]; - const result = await svc.transcribeForProvider(messages, "umans/glm-5.2"); - expect((result[0]?.chunks[0] as { text: string }).text).toContain("no vision-capable model"); + const result = await svc.prepareForProvider(messages, "umans/glm-5.2", { + conversationId: "conv-1", + }); + const text = (result[0]?.chunks[0] as { text: string }).text; + expect(text).toContain("no vision-capable model"); + expect(text).not.toContain("consult_vision"); }); }); -describe("VisionHandoffService.readImageFile", () => { - it("reads the file and transcribes it", async () => { +describe("VisionHandoffService.consultVision", () => { + function makeOrchestratorDouble(response: string): { + orchestrator: NonNullable< + VisionHandoffDeps["resolveOrchestrator"] extends () => infer T ? T : never + >; + handleMessage: ReturnType<typeof vi.fn>; + } { + const handleMessage = vi.fn( + async (input: { + conversationId: string; + text: string; + onEvent: (event: AgentEvent) => void; + }): Promise<void> => { + input.onEvent({ + type: "text-delta", + conversationId: input.conversationId, + turnId: "t1", + delta: response, + }); + input.onEvent({ + type: "done", + conversationId: input.conversationId, + turnId: "t1", + reason: "stop", + }); + }, + ); + return { orchestrator: { handleMessage }, handleMessage }; + } + + it("opens a new consultation with a pasted image and returns convId + response", async () => { const deps = makeDeps(); + const { orchestrator, handleMessage } = makeOrchestratorDouble("The error is on line 12."); + deps.resolveOrchestrator = () => orchestrator; const svc = createVisionHandoffService(deps); - const result = await svc.readImageFile("screenshot.png", "/work"); - expect(deps.readFileAsDataUrl).toHaveBeenCalledWith("screenshot.png", "/work"); - expect(result).toContain("Image analysis"); - expect(result).toContain("FILE(screenshot.png)"); + + // Register an image first (as prepareForProvider would). + const messages: ChatMessage[] = [ + { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,img1" }] }, + ]; + await svc.prepareForProvider(messages, "umans/glm-5.2", { conversationId: "conv-1" }); + + const result = await svc.consultVision("What error is shown?", { + conversationId: "conv-1", + imageIds: [1], + }); + + expect("error" in result).toBe(false); + if (!("error" in result)) { + expect(result.conversationId).toBeTruthy(); + expect(result.response).toContain("line 12"); + expect(result.response).toContain(result.conversationId); + expect(result.response).toContain("dispatch CLI"); + } + // The orchestrator was called with the vision model + the image. + expect(handleMessage).toHaveBeenCalledOnce(); + const call = handleMessage.mock.calls[0]?.[0]; + expect(call.modelName).toBe("umans/kimi-k2.7"); + expect(call.images).toHaveLength(1); + expect(call.images?.[0]?.url).toBe("data:image/png;base64,img1"); + }); + + it("opens a consultation with a file path image", async () => { + const deps = makeDeps(); + const { orchestrator } = makeOrchestratorDouble("It's a diagram."); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + + const result = await svc.consultVision("What is this diagram?", { + conversationId: "conv-1", + path: "diagram.png", + cwd: "/work", + }); + + expect("error" in result).toBe(false); + expect(deps.readFileAsDataUrl).toHaveBeenCalledWith("diagram.png", "/work"); + }); + + it("returns an error when imageId is not registered", async () => { + const deps = makeDeps(); + const { orchestrator } = makeOrchestratorDouble("response"); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + imageIds: [99], // not registered + }); + expect("error" in result).toBe(true); + if ("error" in result) { + expect(result.error).toContain("Image 99"); + } + }); + + it("returns an error when no orchestrator is available", async () => { + const deps = makeDeps(); + // No resolveOrchestrator provided. + const svc = createVisionHandoffService(deps); + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + imageIds: [1], + }); + expect("error" in result).toBe(true); + }); + + it("returns an error when no vision model is available", async () => { + const deps = makeDeps(); + (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]); + const { orchestrator } = makeOrchestratorDouble("response"); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + imageIds: [1], + }); + expect("error" in result).toBe(true); + if ("error" in result) { + expect(result.error).toContain("No vision-capable model"); + } + }); + + it("returns an error when no image source is provided", async () => { + const deps = makeDeps(); + const { orchestrator } = makeOrchestratorDouble("response"); + deps.resolveOrchestrator = () => orchestrator; + const svc = createVisionHandoffService(deps); + const result = await svc.consultVision("What?", { + conversationId: "conv-1", + }); + expect("error" in result).toBe(true); }); }); diff --git a/packages/vision-handoff/src/service.ts b/packages/vision-handoff/src/service.ts index 3f8462a..78f241f 100644 --- a/packages/vision-handoff/src/service.ts +++ b/packages/vision-handoff/src/service.ts @@ -3,40 +3,66 @@ * provider-agnostic vision handoff. * * Two capabilities: - * 1. **Transcription for non-vision models** (`transcribeForProvider`): when a - * user message carries images but the active model cannot see them, this - * calls a vision-capable model (resolved from the catalog — any provider) to - * describe each image, then replaces the image chunks with text. Universal: - * it uses the standard `ProviderContract.stream` interface, never a - * provider-specific vision endpoint. - * 2. **`read_image` tool** (`readImageFile`): reads an image FILE from disk and - * transcribes it via a vision-capable model, returning the text description - * — so any model (vision or not) can analyze an image referenced in code. + * 1. **prepareForProvider** (`prepareForProvider`): when a user message carries + * images but the active model cannot see them, this replaces each image chunk + * with a numbered placeholder (telling the model to call `consult_vision`) + * and registers the image data in a per-conversation registry for tool + * access. Vision-capable models pass through unchanged (images flow natively). + * 2. **consult_vision tool** (`consultVision`): opens a NEW conversation tab with + * a vision-capable model (resolved from the catalog — any provider), attaches + * the image(s) + the model's specific question, waits for the response, and + * returns the conversation ID + the vision model's answer. The model (e.g. + * GLM 5.2) directs the analysis — asking exactly what it needs — instead of + * receiving a pre-emptive generic dump. Follow-up questions go through the + * dispatch CLI (the conversation ID is the bridge), not another tool call. * - * Effects (credential store, provider streaming, filesystem, fetch) are - * injected. The pure decisions live in `pure.ts`. This shell wires them. + * Effects (credential store, orchestrator, filesystem) are injected. The pure + * decisions live in `pure.ts`. This shell wires them. */ import type { CredentialStore } from "@dispatch/credential-store"; import type { + AgentEvent, ChatMessage, Chunk, + ImageInput, Logger, ModelInfo, ProviderContract, - ProviderStreamOptions, } from "@dispatch/kernel"; import { defineService, type ServiceHandle } from "@dispatch/kernel"; import { - buildTranscriptionPrompt, collectTextFromStream, findVisionModelName, + formatConsultResult, + formatImagePlaceholder, formatNoVisionPlaceholder, - formatTranscriptionText, isVisionCapable, } from "./pure.js"; /** + * Minimal orchestrator interface the service needs to start vision consultation + * turns. Defined locally (not imported from session-orchestrator) to avoid a + * compile-time dependency — resolved lazily at runtime via a local handle keyed + * to the same service ID. + */ +export interface OrchestratorForVision { + readonly handleMessage: (input: { + readonly conversationId: string; + readonly text: string; + readonly onEvent: (event: AgentEvent) => void; + readonly modelName?: string; + readonly cwd?: string; + readonly images?: readonly ImageInput[]; + readonly systemPrompt?: string; + }) => Promise<void>; +} + +/** Local handle for the session-orchestrator service (same ID, no import dep). */ +export const orchestratorLocalHandle: ServiceHandle<OrchestratorForVision> = + defineService<OrchestratorForVision>("session-orchestrator/orchestrator"); + +/** * Resolved vision model — a provider + its model id, ready to stream from. */ export interface ResolvedVisionModel { @@ -45,6 +71,12 @@ export interface ResolvedVisionModel { readonly modelName: string; } +/** A registered image (looked up by the consult_vision tool via imageId). */ +interface RegisteredImage { + readonly url: string; + readonly mimeType?: string; +} + /** * Dependencies the service needs — all injected (no ambient state). */ @@ -56,24 +88,24 @@ export interface VisionHandoffDeps { ) => { provider: ProviderContract; model: string } | undefined; /** * Read a file from disk as a base64 data URL. Injected so the shell controls - * the filesystem edge (and tests inject a fake). Returns the data URL, or - * throws on error (the caller surfaces it as a tool error). + * the filesystem edge. Returns the data URL, or throws on error. */ readonly readFileAsDataUrl: (path: string, cwd?: string) => Promise<string>; /** - * Fetch an HTTP(S) URL to a data URL (for http image sources). Injected so - * tests inject a fake. Optional — when absent, HTTP image URLs are passed to - * the vision provider as-is (it fetches them). + * Lazily resolve the session-orchestrator (for starting vision consultation + * turns). Returns `undefined` when not available — `consult_vision` degrades + * with an error. Lazy so activation order doesn't matter. */ - readonly fetchUrlAsDataUrl?: (url: string) => Promise<string>; + readonly resolveOrchestrator?: () => OrchestratorForVision | undefined; + /** Generate a new conversation ID for a consultation. Defaults to crypto.randomUUID. */ + readonly generateId?: () => string; readonly logger?: Logger; } export interface VisionHandoffService { /** * Whether a given model (by catalog name) is vision-capable. Uses the - * credential store's ModelInfo + the name heuristic. Async because ModelInfo - * may require a listModels round-trip (cached by the credential store). + * credential store's ModelInfo + the name heuristic. */ readonly isVisionCapable: (modelName: string | undefined) => Promise<boolean>; @@ -84,43 +116,54 @@ export interface VisionHandoffService { readonly resolveVisionModel: (excludeName?: string) => Promise<ResolvedVisionModel | undefined>; /** - * Transcribe a single image URL to a text description via a vision-capable - * model. Returns the description, or a placeholder string when no vision - * model is available (does NOT throw — callers want graceful degradation). - */ - readonly transcribeImage: ( - imageUrl: string, - userQuestion: string | undefined, - opts?: { readonly signal?: AbortSignal; readonly logger?: Logger }, - ) => Promise<string>; - - /** * Transform a message list for the provider: if the active model is * vision-capable, return messages unchanged (images pass through natively). - * If NOT vision-capable, replace every `image` chunk with a text - * description (transcribed via a vision model — once per unique image URL, - * cached within the call) so a text-only model can still reason about the - * images. Never throws — on failure an image becomes a placeholder note. - * - * The PERSISTED history is NOT modified by this (the caller persists the - * original messages with images); this only transforms what the provider sees. + * If NOT vision-capable, replace every `image` chunk with a numbered + * placeholder (telling the model to call `consult_vision`) and register the + * image data in the per-conversation registry for tool access. The PERSISTED + * history is NOT modified — only what the provider sees. Never throws. */ - readonly transcribeForProvider: ( + readonly prepareForProvider: ( messages: readonly ChatMessage[], currentModelName: string | undefined, - opts?: { readonly signal?: AbortSignal; readonly logger?: Logger }, + opts?: { + readonly conversationId?: string; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, ) => Promise<readonly ChatMessage[]>; /** - * Read an image FILE from disk and transcribe it (the `read_image` tool's - * core). Returns the description text. Throws on filesystem error (the tool - * surfaces it as a tool-error result). + * Look up a registered image by conversation ID + image ID. Returns + * `undefined` when the image isn't registered (e.g. after a server restart). + */ + readonly getRegisteredImage: ( + conversationId: string, + imageId: number, + ) => RegisteredImage | undefined; + + /** + * Open a NEW vision consultation conversation: attach image(s) + the model's + * question to a vision-capable model, wait for the response, and return the + * conversation ID + the vision model's answer. The model drives the analysis + * — it asks exactly what it needs. Follow-ups go through the dispatch CLI. + * + * @returns The conversation ID + the vision model's response text, or an + * error string (never throws — the tool surfaces it). */ - readonly readImageFile: ( - path: string, - cwd: string | undefined, - opts?: { readonly signal?: AbortSignal; readonly logger?: Logger }, - ) => Promise<string>; + readonly consultVision: ( + question: string, + opts: { + readonly conversationId: string; + readonly imageIds?: readonly number[]; + readonly path?: string; + readonly cwd?: string; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, + ) => Promise< + { readonly conversationId: string; readonly response: string } | { readonly error: string } + >; } export const visionHandoffHandle: ServiceHandle<VisionHandoffService> = @@ -133,6 +176,12 @@ function hasImageChunks(messages: readonly ChatMessage[]): boolean { export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHandoffService { const log = deps.logger; + const generateId = deps.generateId ?? (() => crypto.randomUUID()); + + // Per-conversation image registry: conversationId → (imageId → image data). + // Populated by prepareForProvider; consulted by the consult_vision tool. + // In-memory only (cleared on restart — the user re-pastes if needed). + const imageRegistry = new Map<string, Map<number, RegisteredImage>>(); async function getInfo(modelName: string): Promise<ModelInfo | undefined> { return deps.credentialStore.getModelInfo(modelName); @@ -149,41 +198,6 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando return { provider: resolved.provider, model: resolved.model, modelName: name }; } - async function streamVisionText( - vision: ResolvedVisionModel, - imageUrl: string, - prompt: string, - opts?: { readonly signal?: AbortSignal; readonly logger?: Logger }, - ): Promise<string> { - // Build a single-turn user message: [text prompt, image]. The vision model - // receives the image natively via the OpenAI-compatible content array - // (convertMessages serializes the image chunk to image_url). - const userMessage: ChatMessage = { - role: "user", - chunks: [ - { type: "text", text: prompt }, - { type: "image", url: imageUrl }, - ], - }; - const providerOpts: ProviderStreamOptions = { - model: vision.model, - // NOTE: temperature is deliberately OMITTED. Different vision providers - // have different constraints (e.g. Moonshot/Kimi only allows temperature: - // 1; others allow 0–2). Hardcoding any value risks an HTTP 400 from a - // provider that rejects it. Omitting lets each provider use its own - // default — the truly universal, provider-agnostic choice. - // A short system prompt keeps the vision model focused on describing. - systemPrompt: - "You are a vision assistant. Describe images faithfully and thoroughly for a developer who cannot see them.", - }; - const streamOpts: Parameters<ProviderContract["stream"]>[2] = { - ...providerOpts, - ...(opts?.logger !== undefined ? { logger: opts.logger } : {}), - }; - const stream = vision.provider.stream([userMessage], [], streamOpts); - return collectTextFromStream(stream); - } - const service: VisionHandoffService = { async isVisionCapable(modelName: string | undefined): Promise<boolean> { if (modelName === undefined) return false; @@ -193,35 +207,14 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando resolveVisionModel, - async transcribeImage( - imageUrl: string, - userQuestion: string | undefined, - opts?: { readonly signal?: AbortSignal; readonly logger?: Logger }, - ): Promise<string> { - const vision = await resolveVisionModel(); - if (vision === undefined) { - log?.warn("vision-handoff: no vision-capable model available for transcription"); - return formatNoVisionPlaceholder(); - } - const prompt = buildTranscriptionPrompt(userQuestion); - try { - const description = await streamVisionText(vision, imageUrl, prompt, opts); - const trimmed = description.trim(); - if (trimmed.length === 0) { - return "[Image analysis produced no output.]"; - } - return formatTranscriptionText(trimmed, vision.modelName); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - log?.warn("vision-handoff: transcription failed", { error: msg }); - return `[Image analysis failed: ${msg}]`; - } - }, - - async transcribeForProvider( + async prepareForProvider( messages: readonly ChatMessage[], currentModelName: string | undefined, - opts?: { readonly signal?: AbortSignal; readonly logger?: Logger }, + opts?: { + readonly conversationId?: string; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, ): Promise<readonly ChatMessage[]> { // Fast path: no images anywhere → nothing to do. if (!hasImageChunks(messages)) return messages; @@ -232,35 +225,41 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando if (capable) return messages; } - // Non-vision model: transcribe each unique image URL once (cached). - const cache = new Map<string, string>(); - const userText = messages - .filter((m) => m.role === "user") - .flatMap((m) => m.chunks) - .filter((c): c is { type: "text"; text: string } => c.type === "text") - .map((c) => c.text) - .join(" "); - - async function transcribeCached(url: string): Promise<string> { - const cached = cache.get(url); - if (cached !== undefined) return cached; - const description = await service.transcribeImage(url, userText, opts); - cache.set(url, description); - return description; - } + // Non-vision model: check if a vision model is available at all. + const vision = await resolveVisionModel(); + const convId = opts?.conversationId; + const placeholderFn = + vision !== undefined && convId !== undefined + ? (id: number) => formatImagePlaceholder(id) + : () => formatNoVisionPlaceholder(); + + // Replace each image chunk with a numbered placeholder. Assign sequential + // 1-based IDs across all messages and register each image in the + // per-conversation registry so the consult_vision tool can look it up. + let seqId = 0; const result: ChatMessage[] = []; for (const msg of messages) { if (!msg.chunks.some((c) => c.type === "image")) { result.push(msg); continue; } - // Replace image chunks with transcribed text chunks; keep all else. const newChunks: Chunk[] = []; for (const chunk of msg.chunks) { if (chunk.type === "image") { - const description = await transcribeCached(chunk.url); - newChunks.push({ type: "text", text: description }); + seqId++; + if (convId !== undefined && vision !== undefined) { + let convImages = imageRegistry.get(convId); + if (convImages === undefined) { + convImages = new Map(); + imageRegistry.set(convId, convImages); + } + convImages.set(seqId, { + url: chunk.url, + ...(chunk.mimeType !== undefined ? { mimeType: chunk.mimeType } : {}), + }); + } + newChunks.push({ type: "text", text: placeholderFn(seqId) }); } else { newChunks.push(chunk); } @@ -270,13 +269,109 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando return result; }, - async readImageFile( - path: string, - cwd: string | undefined, - opts?: { readonly signal?: AbortSignal; readonly logger?: Logger }, - ): Promise<string> { - const dataUrl = await deps.readFileAsDataUrl(path, cwd); - return service.transcribeImage(dataUrl, undefined, opts); + getRegisteredImage(conversationId: string, imageId: number): RegisteredImage | undefined { + return imageRegistry.get(conversationId)?.get(imageId); + }, + + async consultVision( + question: string, + opts: { + readonly conversationId: string; + readonly imageIds?: readonly number[]; + readonly path?: string; + readonly cwd?: string; + readonly signal?: AbortSignal; + readonly logger?: Logger; + }, + ): Promise< + { readonly conversationId: string; readonly response: string } | { readonly error: string } + > { + const orchestrator = deps.resolveOrchestrator?.(); + if (orchestrator === undefined) { + return { + error: "The session orchestrator is not available — cannot start a vision consultation.", + }; + } + + const vision = await resolveVisionModel(); + if (vision === undefined) { + return { + error: + "No vision-capable model is available in the catalog. Install or configure one (e.g. kimi) to enable image analysis.", + }; + } + + // Collect image data URLs to attach. + const images: ImageInput[] = []; + if (opts.imageIds !== undefined) { + for (const id of opts.imageIds) { + const img = service.getRegisteredImage(opts.conversationId, id); + if (img === undefined) { + return { + error: `Image ${id} is not registered. It may have been lost after a server restart — ask the user to re-paste the image.`, + }; + } + images.push({ + url: img.url, + ...(img.mimeType !== undefined ? { mimeType: img.mimeType } : {}), + }); + } + } + if (opts.path !== undefined) { + try { + const dataUrl = await deps.readFileAsDataUrl(opts.path, opts.cwd); + images.push({ url: dataUrl }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { error: `Failed to read image file "${opts.path}": ${msg}` }; + } + } + if (images.length === 0) { + return { + error: + "No image to consult about. Provide imageIds (for pasted images) or path (for a file).", + }; + } + + // Start a NEW conversation with the vision model. + const consultationId = generateId(); + log?.info("vision-handoff: starting consultation", { + consultationId, + visionModel: vision.modelName, + imageCount: images.length, + fromConversation: opts.conversationId, + }); + + let responseText = ""; + let errorMessage = ""; + try { + await orchestrator.handleMessage({ + conversationId: consultationId, + text: question, + images, + modelName: vision.modelName, + ...(opts.cwd !== undefined ? { cwd: opts.cwd } : {}), + systemPrompt: + "You are a vision assistant. A developer who cannot see images is asking you specific questions about an image they attached. Answer their question precisely and thoroughly.", + onEvent: (event: AgentEvent) => { + if (event.type === "text-delta") { + responseText += event.delta; + } else if (event.type === "error") { + errorMessage = event.message; + } + }, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { error: `Vision consultation failed: ${msg}` }; + } + + if (errorMessage.length > 0 && responseText.trim().length === 0) { + return { error: `Vision consultation failed: ${errorMessage}` }; + } + + const response = formatConsultResult(consultationId, responseText); + return { conversationId: consultationId, response }; }, }; diff --git a/packages/vision-handoff/src/tool.ts b/packages/vision-handoff/src/tool.ts index 3995598..86be2ed 100644 --- a/packages/vision-handoff/src/tool.ts +++ b/packages/vision-handoff/src/tool.ts @@ -1,65 +1,134 @@ /** - * read_image tool — lets any model (vision-capable or not) analyze an image - * FILE on disk by handing it off to a vision-capable model. + * consult_vision tool — lets any model (vision-capable or not) consult a + * vision-capable model about an image by opening a NEW conversation tab. * - * The tool reads the image file into a base64 data URL, then asks the vision - * handoff service to transcribe it (via a vision-capable model resolved from - * the catalog) and returns the textual description as the tool result. This is - * the universal mechanism: it works regardless of whether the active model has - * vision, because the result is plain text the model reasons about. + * The tool attaches image(s) + the model's specific question to a vision-capable + * model (resolved from the catalog — e.g. Kimi), waits for the response, and + * returns the conversation ID + the vision model's answer. The MODEL directs the + * analysis — it asks exactly what it needs to know — instead of receiving a + * pre-emptive generic dump. * - * For images PASTED into the chat, the orchestrator's auto-transcription handles - * them (no tool call needed). This tool is for images REFERENCED IN CODE by path - * (e.g. a screenshot, diagram, or mockup the model discovered while reading files). + * For images PASTED into the chat, the model references them by `imageIds` (from + * the "[Image N attached]" placeholders the orchestrator injected). For image + * FILES on disk, the model passes a `path`. + * + * Follow-up questions are NOT handled by this tool — the model uses the dispatch + * CLI to continue the vision conversation (the returned conversation ID is the + * bridge; the model can load the `dispatch-cli` skill for the exact commands). */ import type { ToolContract, ToolExecuteContext, ToolResult } from "@dispatch/kernel"; import type { VisionHandoffService } from "./service.js"; -export function createReadImageTool(service: VisionHandoffService): ToolContract { +export function createConsultVisionTool(service: VisionHandoffService): ToolContract { return { - name: "read_image", + name: "consult_vision", description: - "Read and analyze an image file on disk (PNG, JPEG, WebP, GIF). Returns a " + - "detailed textual description of the image's contents — useful when you " + - "encounter a screenshot, diagram, UI mockup, or chart referenced in the " + - "codebase and need to understand what it shows. The analysis is performed " + - "by a vision-capable model, so you can use this even if you cannot " + - "directly view images. Pass a file path (relative to the cwd or absolute).", + "Consult a vision-capable model (e.g. Kimi) about an image by opening a new " + + "conversation tab. Attaches the image(s) + your specific question, waits for " + + "the vision model's response, and returns the conversation ID + the answer. " + + "Use this when you cannot view an image (e.g. a pasted screenshot or diagram) " + + "and need to know what it shows — ask a SPECIFIC question (e.g. 'What error " + + "message is on line 12?' rather than 'describe this image'). The conversation " + + "ID is returned so follow-up questions can be asked via the dispatch CLI.", parameters: { type: "object", properties: { + question: { + type: "string", + description: + "Your specific question about the image. Be precise — the vision model " + + "will answer exactly this. E.g. 'What error message is displayed?' or " + + "'Compare the layout of these two screenshots.'", + }, + imageIds: { + type: "array", + items: { type: "number" }, + description: + "The IDs of pasted images to attach (from the '[Image N attached]' " + + "placeholders in the conversation). Pass multiple to attach several " + + "images to one consultation (e.g. [1, 2] to compare them).", + }, path: { type: "string", description: - "Path to the image file to analyze. Relative paths resolve against " + - "the conversation's working directory; absolute paths are used as-is.", + "Path to an image FILE on disk to attach (alternative to imageIds for " + + "code-referenced images). Relative paths resolve against the cwd.", }, }, - required: ["path"], + required: ["question"], }, concurrencySafe: true, async execute(args: unknown, ctx: ToolExecuteContext): Promise<ToolResult> { - const input = args as { path?: unknown } | null; + const input = args as { + question?: unknown; + imageIds?: unknown; + path?: unknown; + } | null; + + const question = input?.question; + if (typeof question !== "string" || question.trim().length === 0) { + return { + content: "Error: 'question' is required and must be a non-empty string.", + isError: true, + }; + } + + const imageIds = input?.imageIds; const path = input?.path; - if (typeof path !== "string" || path.trim().length === 0) { + + // Parse imageIds (must be an array of numbers if present). + let parsedImageIds: number[] | undefined; + if (imageIds !== undefined) { + if (!Array.isArray(imageIds)) { + return { content: "Error: 'imageIds' must be an array of numbers.", isError: true }; + } + parsedImageIds = imageIds.filter((n): n is number => typeof n === "number"); + if (parsedImageIds.length === 0) { + return { content: "Error: 'imageIds' must contain at least one number.", isError: true }; + } + } + + // path must be a string if present. + let parsedPath: string | undefined; + if (path !== undefined) { + if (typeof path !== "string" || path.trim().length === 0) { + return { content: "Error: 'path' must be a non-empty string.", isError: true }; + } + parsedPath = path; + } + + // At least one image source is required. + if (parsedImageIds === undefined && parsedPath === undefined) { return { - content: "Error: 'path' is required and must be a non-empty string.", + content: + "Error: provide 'imageIds' (for pasted images) or 'path' (for a file) " + + "to attach an image to the consultation.", isError: true, }; } - const span = ctx.log.span("read_image.execute", { path }); + + const span = ctx.log.span("consult_vision.execute", { + imageCount: (parsedImageIds?.length ?? 0) + (parsedPath !== undefined ? 1 : 0), + }); try { - const description = await service.readImageFile(path, ctx.cwd, { + const result = await service.consultVision(question, { + conversationId: ctx.conversationId ?? "", + ...(parsedImageIds !== undefined ? { imageIds: parsedImageIds } : {}), + ...(parsedPath !== undefined ? { path: parsedPath } : {}), + ...(ctx.cwd !== undefined ? { cwd: ctx.cwd } : {}), signal: ctx.signal, logger: ctx.log, }); - span.end({ attrs: { descriptionLength: description.length } }); - return { content: description }; + span.end({ attrs: { ok: !("error" in result) } }); + if ("error" in result) { + return { content: result.error, isError: true }; + } + return { content: result.response }; } catch (err: unknown) { span.end({ err }); return { - content: `Error reading image: ${err instanceof Error ? err.message : String(err)}`, + content: `Error during vision consultation: ${err instanceof Error ? err.message : String(err)}`, isError: true, }; } |
