summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-27 18:12:04 +0900
committerAdam Malczewski <[email protected]>2026-06-27 18:12:04 +0900
commite76790fef11de3cc33f60689f9301030ed740cd6 (patch)
tree42032276165ec91ec9eebb27738165296729953f
parent72d08ddffbbf70d73db8d223aac20937f662560f (diff)
downloaddispatch-e76790fef11de3cc33f60689f9301030ed740cd6.tar.gz
dispatch-e76790fef11de3cc33f60689f9301030ed740cd6.zip
feat(vision-handoff): model-directed consult_vision tool replacing auto-transcription
-rw-r--r--packages/session-orchestrator/src/orchestrator.ts17
-rw-r--r--packages/vision-handoff/src/extension.ts67
-rw-r--r--packages/vision-handoff/src/index.ts8
-rw-r--r--packages/vision-handoff/src/pure.test.ts51
-rw-r--r--packages/vision-handoff/src/pure.ts85
-rw-r--r--packages/vision-handoff/src/service.test.ts252
-rw-r--r--packages/vision-handoff/src/service.ts371
-rw-r--r--packages/vision-handoff/src/tool.ts127
8 files changed, 623 insertions, 355 deletions
diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts
index ac1eaf4..4f4bb3e 100644
--- a/packages/session-orchestrator/src/orchestrator.ts
+++ b/packages/session-orchestrator/src/orchestrator.ts
@@ -43,16 +43,20 @@ import type { ToolAssembly } from "./tools-filter.js";
* off cleanly when the extension isn't loaded (images pass through unchanged,
* which is correct for vision-capable models and a no-op for text-only turns).
*
- * `transcribeForProvider` transforms a message list for the provider: if the
+ * `prepareForProvider` transforms a message list for the provider: if the
* active model is vision-capable, messages pass through unchanged; otherwise
- * image chunks are replaced with text descriptions (transcribed via a
- * vision-capable model). Never throws — degrades to placeholders.
+ * image chunks are replaced with numbered placeholders (telling the model to
+ * call `consult_vision`) and the images are registered for tool access.
*/
export interface VisionHandoffService {
- readonly transcribeForProvider: (
+ readonly prepareForProvider: (
messages: readonly ChatMessage[],
currentModelName: string | undefined,
- opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
+ opts?: {
+ readonly conversationId?: string;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
) => Promise<readonly ChatMessage[]>;
}
@@ -772,10 +776,11 @@ export function createSessionOrchestrator(
const visionHandoff = deps.resolveVisionHandoff?.();
let providerMessages: readonly ChatMessage[] = [...history, userMsg];
if (visionHandoff !== undefined) {
- providerMessages = await visionHandoff.transcribeForProvider(
+ providerMessages = await visionHandoff.prepareForProvider(
providerMessages,
effectiveModelName,
{
+ conversationId,
signal: controller.signal,
...(turnLogger !== undefined ? { logger: turnLogger } : {}),
},
diff --git a/packages/vision-handoff/src/extension.ts b/packages/vision-handoff/src/extension.ts
index aa745b7..2f75a6b 100644
--- a/packages/vision-handoff/src/extension.ts
+++ b/packages/vision-handoff/src/extension.ts
@@ -1,15 +1,17 @@
/**
* vision-handoff extension — registers the universal vision handoff service +
- * the `read_image` tool.
+ * the `consult_vision` tool.
*
- * The service performs provider-agnostic vision handoff: it resolves a
- * vision-capable model from the catalog (any provider), streams an image to it
- * via the standard `ProviderContract.stream` interface, and folds the textual
- * description back — so a non-vision model (e.g. glm-5.2) can still reason about
- * images, and any model can analyze image FILES referenced in code.
+ * The service performs provider-agnostic vision handoff: when a non-vision model
+ * (e.g. glm-5.2) receives an image, it replaces the image with a numbered
+ * placeholder and registers it for tool access. The `consult_vision` tool opens
+ * a NEW conversation tab with a vision-capable model (e.g. Kimi), attaches the
+ * image + the model's specific question, and returns the conversation ID + the
+ * vision model's answer. Follow-ups go through the dispatch CLI.
*
- * Effects (filesystem, fetch) live here in the shell, injected into the service.
- * The pure decisions live in `pure.ts`. No `console.*`; logging via `host.logger`.
+ * Effects (filesystem, orchestrator) live here in the shell, injected into the
+ * service. The pure decisions live in `pure.ts`. No `console.*`; logging via
+ * `host.logger`.
*/
import { readFile } from "node:fs/promises";
@@ -17,8 +19,12 @@ import { extname, isAbsolute, resolve as pathResolve } from "node:path";
import type { CredentialStore } from "@dispatch/credential-store";
import { credentialStoreHandle } from "@dispatch/credential-store";
import type { Extension, HostAPI, Manifest } from "@dispatch/kernel";
-import { createVisionHandoffService, visionHandoffHandle } from "./service.js";
-import { createReadImageTool } from "./tool.js";
+import {
+ createVisionHandoffService,
+ orchestratorLocalHandle,
+ visionHandoffHandle,
+} from "./service.js";
+import { createConsultVisionTool } from "./tool.js";
export const manifest: Manifest = {
id: "vision-handoff",
@@ -28,7 +34,7 @@ export const manifest: Manifest = {
trust: "bundled",
activation: "eager",
capabilities: { network: true },
- contributes: { services: ["vision-handoff/service"], tools: ["read_image"] },
+ contributes: { services: ["vision-handoff/service"], tools: ["consult_vision"] },
};
/** MIME types for recognized image extensions. */
@@ -54,30 +60,11 @@ async function readFileAsDataUrl(path: string, cwd?: string): Promise<string> {
return `data:${mime};base64,${buf.toString("base64")}`;
}
-/**
- * Fetch an HTTP(S) image URL and convert it to a base64 data URL (so it can be
- * sent to the vision model inline, regardless of whether the provider can fetch
- * remote URLs). The shell edge — real `globalThis.fetch`.
- */
-async function fetchUrlAsDataUrl(url: string): Promise<string> {
- const res = await fetch(url);
- if (!res.ok) {
- throw new Error(`Failed to fetch image: HTTP ${res.status}`);
- }
- const buf = new Uint8Array(await res.arrayBuffer());
- const mime = res.headers.get("content-type") ?? "image/png";
- // Buffer/base64 in Bun + Node. Convert byte-by-byte without non-null asserts.
- let binary = "";
- for (const byte of buf) binary += String.fromCharCode(byte);
- const base64 = btoa(binary);
- return `data:${mime};base64,${base64}`;
-}
-
export async function activate(host: HostAPI): Promise<void> {
const credentialStore = host.getService(credentialStoreHandle) as CredentialStore | undefined;
if (credentialStore === undefined) {
host.logger.warn(
- "vision-handoff: credential-store service not available. The read_image tool and image transcription are disabled.",
+ "vision-handoff: credential-store service not available. The consult_vision tool and image handoff are disabled.",
);
return;
}
@@ -94,13 +81,25 @@ export async function activate(host: HostAPI): Promise<void> {
credentialStore,
resolveModel,
readFileAsDataUrl,
- fetchUrlAsDataUrl,
+ // Lazily resolve the session-orchestrator (for starting vision consultation
+ // turns). By the time consult_vision is called at runtime, all extensions
+ // have activated. The activated-manifests guard avoids a getService throw
+ // when the orchestrator isn't loaded.
+ resolveOrchestrator: () => {
+ const loaded = host.getExtensions().some((m) => m.id === "session-orchestrator");
+ if (!loaded) return undefined;
+ try {
+ return host.getService(orchestratorLocalHandle);
+ } catch {
+ return undefined;
+ }
+ },
logger: host.logger.child({ extensionId: "vision-handoff" }),
});
host.provideService(visionHandoffHandle, service);
- host.defineTool(createReadImageTool(service));
- host.logger.info("vision-handoff: registered (read_image tool + transcription service)");
+ host.defineTool(createConsultVisionTool(service));
+ host.logger.info("vision-handoff: registered (consult_vision tool + handoff service)");
}
export const extension: Extension = { manifest, activate };
diff --git a/packages/vision-handoff/src/index.ts b/packages/vision-handoff/src/index.ts
index 4a13e65..2713346 100644
--- a/packages/vision-handoff/src/index.ts
+++ b/packages/vision-handoff/src/index.ts
@@ -1,19 +1,21 @@
export { extension, manifest } from "./extension.js";
export {
- buildTranscriptionPrompt,
collectTextFromStream,
findVisionModelName,
+ formatConsultResult,
+ formatImagePlaceholder,
formatNoVisionPlaceholder,
- formatTranscriptionText,
isVisionCapable,
} from "./pure.js";
export type {
+ OrchestratorForVision,
ResolvedVisionModel,
VisionHandoffDeps,
VisionHandoffService,
} from "./service.js";
export {
createVisionHandoffService,
+ orchestratorLocalHandle,
visionHandoffHandle,
} from "./service.js";
-export { createReadImageTool } from "./tool.js";
+export { createConsultVisionTool } from "./tool.js";
diff --git a/packages/vision-handoff/src/pure.test.ts b/packages/vision-handoff/src/pure.test.ts
index 89dac72..4198d00 100644
--- a/packages/vision-handoff/src/pure.test.ts
+++ b/packages/vision-handoff/src/pure.test.ts
@@ -1,11 +1,11 @@
import type { ModelInfo, ProviderEvent } from "@dispatch/kernel";
import { describe, expect, it } from "vitest";
import {
- buildTranscriptionPrompt,
collectTextFromStream,
findVisionModelName,
+ formatConsultResult,
+ formatImagePlaceholder,
formatNoVisionPlaceholder,
- formatTranscriptionText,
isVisionCapable,
} from "./pure.js";
@@ -43,7 +43,7 @@ describe("findVisionModelName", () => {
return map[name];
};
- it("finds the first kimi-family model via name heuristic (no async lookup needed)", async () => {
+ it("finds the first kimi-family model via name heuristic", async () => {
const name = await findVisionModelName(
["umans/glm-5.2", "umans/kimi-k2.7", "umans/llama-vision"],
getInfo,
@@ -90,7 +90,7 @@ describe("collectTextFromStream", () => {
expect(text).toBe("Hello world!");
});
- it("ignores non-text events (reasoning, usage, tool-call, finish)", async () => {
+ it("ignores non-text events", async () => {
const events: ProviderEvent[] = [
{ type: "reasoning-delta", delta: "thinking..." },
{ type: "text-delta", delta: "answer" },
@@ -115,27 +115,38 @@ describe("collectTextFromStream", () => {
});
});
-describe("prompt + formatting helpers", () => {
- it("buildTranscriptionPrompt includes focus when a question is given", () => {
- const prompt = buildTranscriptionPrompt("What error is shown?");
- expect(prompt).toContain("Describe this image in detail");
- expect(prompt).toContain('The user asked: "What error is shown?"');
+describe("formatImagePlaceholder", () => {
+ it("includes the image ID and mentions consult_vision", () => {
+ const text = formatImagePlaceholder(1);
+ expect(text).toContain("Image 1");
+ expect(text).toContain("consult_vision");
+ expect(text).toContain("imageIds=[1]");
});
- it("buildTranscriptionPrompt omits focus when no question", () => {
- const prompt = buildTranscriptionPrompt(undefined);
- expect(prompt).toContain("Describe this image in detail");
- expect(prompt).not.toContain("The user asked");
- });
-
- it("formatTranscriptionText names the vision model", () => {
- expect(formatTranscriptionText("a red car", "umans/kimi-k2.7")).toBe(
- "[Image analysis (via umans/kimi-k2.7)]: a red car",
- );
+ it("increments the ID for each image", () => {
+ expect(formatImagePlaceholder(2)).toContain("Image 2");
+ expect(formatImagePlaceholder(2)).toContain("imageIds=[2]");
});
+});
- it("formatNoVisionPlaceholder explains the limitation", () => {
+describe("formatNoVisionPlaceholder", () => {
+ it("explains the limitation", () => {
const text = formatNoVisionPlaceholder();
expect(text).toContain("no vision-capable model");
});
});
+
+describe("formatConsultResult", () => {
+ it("includes the conversation ID, the response, and the dispatch CLI hint", () => {
+ const result = formatConsultResult("abc-123", "The error is on line 12.");
+ expect(result).toContain("abc-123");
+ expect(result).toContain("The error is on line 12.");
+ expect(result).toContain("dispatch CLI");
+ });
+
+ it("trims the response", () => {
+ const result = formatConsultResult("c1", " spaced ");
+ expect(result).toContain("spaced");
+ expect(result).not.toContain("spaced ");
+ });
+});
diff --git a/packages/vision-handoff/src/pure.ts b/packages/vision-handoff/src/pure.ts
index 11eeefc..5eeb1a3 100644
--- a/packages/vision-handoff/src/pure.ts
+++ b/packages/vision-handoff/src/pure.ts
@@ -2,9 +2,10 @@
* Pure decision helpers for the vision handoff.
*
* No I/O, no ambient state. The shell (the extension + the service) injects the
- * effects (credential store lookups, provider streaming). This module owns only
- * the policy: which model is vision-capable, how to build a transcription
- * request, and how to fold a provider's streamed text into a description.
+ * effects (credential store lookups, orchestrator, provider streaming). This
+ * module owns only the policy: which model is vision-capable, how to format
+ * image placeholders for non-vision models, and how to format the
+ * consultation tool's result.
*/
import type { ModelInfo, ProviderEvent } from "@dispatch/kernel";
@@ -36,9 +37,7 @@ export function isVisionCapable(
/**
* Find the first vision-capable model name in a catalog, given a lookup that
* resolves a `<credentialName>/<model>` → `ModelInfo`. Returns `undefined` when
- * no vision-capable model is available (the handoff degrades: images are
- * replaced with a placeholder note). Pure given the (async) lookup — no
- * ambient state, no side effects.
+ * no vision-capable model is available. Pure given the (async) lookup.
*
* @param catalog The full list of model names (`<credentialName>/<model>`).
* @param getInfo Async lookup of a model name → ModelInfo (from the credential store).
@@ -52,8 +51,7 @@ export async function findVisionModelName(
for (const name of catalog) {
if (exclude !== undefined && name === exclude) continue;
// Fast path: the name heuristic lets us short-circuit without an async
- // lookup for known vision families (kimi). This avoids a round-trip to
- // listModels for the common case.
+ // lookup for known vision families (kimi).
const slash = name.indexOf("/");
const modelId = slash >= 0 ? name.slice(slash + 1) : name;
if (isVisionModelId(modelId)) return name;
@@ -64,11 +62,10 @@ export async function findVisionModelName(
}
/**
- * Fold a provider's streamed events into a single text string (the
- * transcription). Pure given the async iterable — collects `text-delta` events,
- * ignores everything else (reasoning, usage, tool-calls, errors). If the stream
- * yields an error event, it is surfaced as a thrown Error so the caller can
- * decide how to degrade (placeholder vs. fail). Pure: input → output, no I/O.
+ * Fold a provider's streamed events into a single text string. Pure given the
+ * async iterable — collects `text-delta` events, ignores everything else
+ * (reasoning, usage, tool-calls). If the stream yields an error event, it is
+ * surfaced as a thrown Error so the caller can decide how to degrade.
*/
export async function collectTextFromStream(stream: AsyncIterable<ProviderEvent>): Promise<string> {
let text = "";
@@ -83,43 +80,26 @@ export async function collectTextFromStream(stream: AsyncIterable<ProviderEvent>
}
/**
- * Build the prompt sent to the vision model to transcribe an image. Kept here
- * (pure) so the prompt is testable and stable. The prompt asks for a thorough
- * description so the text-only model has enough detail to reason about the
- * image's contents. Pure.
+ * Format the placeholder text that replaces an `image` chunk when a non-vision
+ * model is active. The placeholder tells the model an image is attached and it
+ * should call `consult_vision` to analyze it — the model drives the analysis
+ * (asking a specific question) rather than receiving a pre-emptive generic dump.
*
- * @param userQuestion The user's own message text (may be empty) — passed so
- * the vision model can tailor its description to what the user actually asked.
+ * @param imageId The 1-based ID assigned to this image (used by the tool to
+ * look up the registered image data).
+ * Pure.
*/
-export function buildTranscriptionPrompt(userQuestion: string | undefined): string {
- const focus =
- userQuestion && userQuestion.trim().length > 0
- ? `\n\nThe user asked: "${userQuestion.trim()}". Focus your description on what is relevant to that question, but still describe the whole image.`
- : "";
+export function formatImagePlaceholder(imageId: number): string {
return (
- "Describe this image in detail. Include: the overall scene/subject, " +
- "visible text (transcribe verbatim), key objects, layout, colors, and any " +
- "notable details a developer or user would need to understand the image." +
- focus
+ `[Image ${imageId} attached — you cannot view images. Call the ` +
+ `consult_vision tool with imageIds=[${imageId}] and a specific question ` +
+ `to analyze it via a vision-capable model.]`
);
}
/**
- * Format a single image's transcription as a text chunk string for the
- * persisted user message. The note names the vision model so the consumer knows
- * the description's provenance. Pure.
- */
-export function formatTranscriptionText(
- description: string,
- visionModelName: string | undefined,
-): string {
- const source = visionModelName ?? "vision model";
- return `[Image analysis (via ${source})]: ${description}`;
-}
-
-/**
* Placeholder text used when NO vision-capable model is available (the
- * degraded path). Pure.
+ * degraded path — the tool cannot function). Pure.
*/
export function formatNoVisionPlaceholder(): string {
return (
@@ -127,3 +107,24 @@ export function formatNoVisionPlaceholder(): string {
"Install or configure a vision-capable model (e.g. kimi) to enable image analysis.]"
);
}
+
+/**
+ * Format the `consult_vision` tool's result string. Returns the conversation ID
+ * (so the model / user can continue the vision consultation), the vision model's
+ * response, and a note that follow-up questions use the dispatch CLI (the model
+ * can load the `dispatch-cli` skill for the exact commands).
+ *
+ * Pure.
+ *
+ * @param conversationId The new vision consultation conversation ID.
+ * @param response The vision model's answer to the model's question.
+ */
+export function formatConsultResult(conversationId: string, response: string): string {
+ const trimmed = response.trim();
+ return (
+ `Vision consultation opened in conversation ${conversationId}.\n\n` +
+ `Response: ${trimmed}\n\n` +
+ `To ask follow-up questions about this image, use the dispatch CLI ` +
+ `(conversation: ${conversationId}).`
+ );
+}
diff --git a/packages/vision-handoff/src/service.test.ts b/packages/vision-handoff/src/service.test.ts
index fe99d17..73c647b 100644
--- a/packages/vision-handoff/src/service.test.ts
+++ b/packages/vision-handoff/src/service.test.ts
@@ -1,9 +1,9 @@
import type {
+ AgentEvent,
ChatMessage,
ModelInfo,
ProviderContract,
ProviderEvent,
- ProviderStreamOptions,
ToolContract,
} from "@dispatch/kernel";
import { describe, expect, it, vi } from "vitest";
@@ -21,7 +21,6 @@ function makeVisionProvider(
(
messages: readonly ChatMessage[],
_tools: readonly ToolContract[],
- _opts?: ProviderStreamOptions,
): AsyncIterable<ProviderEvent> => {
const img = messages.flatMap((m) => m.chunks).find((c) => c.type === "image");
const url = img && img.type === "image" ? img.url : "";
@@ -91,46 +90,11 @@ describe("VisionHandoffService.resolveVisionModel", () => {
it("excludes the given model", async () => {
const svc = createVisionHandoffService(makeDeps());
const vision = await svc.resolveVisionModel("umans/kimi-k2.7");
- // kimi is the only vision model; excluding it → undefined.
expect(vision).toBeUndefined();
});
});
-describe("VisionHandoffService.transcribeImage", () => {
- it("returns a formatted description from the vision model", async () => {
- const svc = createVisionHandoffService(makeDeps());
- const result = await svc.transcribeImage("data:image/png;base64,xxx", "what is this?");
- expect(result).toBe(
- "[Image analysis (via umans/kimi-k2.7)]: DESCRIPTION of data:image/png;base64,xxx",
- );
- });
-
- it("returns a placeholder when no vision model is available", async () => {
- const deps = makeDeps();
- // Empty catalog → no vision model.
- (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]);
- const svc = createVisionHandoffService(deps);
- const result = await svc.transcribeImage("data:image/png;base64,xxx", undefined);
- expect(result).toContain("no vision-capable model");
- });
-
- it("returns an error note when the vision stream errors", async () => {
- const errorProvider: ProviderContract = {
- id: "umans",
- stream: vi.fn(async function* (): AsyncIterable<ProviderEvent> {
- yield { type: "error", message: "vision API down" };
- }),
- };
- const deps = makeDeps({
- resolveModel: vi.fn(() => ({ provider: errorProvider, model: "kimi-k2.7" })),
- });
- const svc = createVisionHandoffService(deps);
- const result = await svc.transcribeImage("data:image/png;base64,xxx", undefined);
- expect(result).toContain("Image analysis failed: vision API down");
- });
-});
-
-describe("VisionHandoffService.transcribeForProvider", () => {
+describe("VisionHandoffService.prepareForProvider", () => {
it("passes messages through unchanged when the model is vision-capable", async () => {
const deps = makeDeps();
const svc = createVisionHandoffService(deps);
@@ -143,19 +107,19 @@ describe("VisionHandoffService.transcribeForProvider", () => {
],
},
];
- const result = await svc.transcribeForProvider(messages, "umans/kimi-k2.7");
- expect(result).toBe(messages); // same reference — no copy, no transcription
+ const result = await svc.prepareForProvider(messages, "umans/kimi-k2.7");
+ expect(result).toBe(messages); // same reference — no copy, no change
});
it("passes messages through unchanged when there are no images", async () => {
const deps = makeDeps();
const svc = createVisionHandoffService(deps);
const messages: ChatMessage[] = [{ role: "user", chunks: [{ type: "text", text: "hi" }] }];
- const result = await svc.transcribeForProvider(messages, "umans/glm-5.2");
+ const result = await svc.prepareForProvider(messages, "umans/glm-5.2");
expect(result).toBe(messages);
});
- it("transcribes image chunks to text for a non-vision model", async () => {
+ it("replaces image chunks with numbered placeholders for a non-vision model", async () => {
const deps = makeDeps();
const svc = createVisionHandoffService(deps);
const messages: ChatMessage[] = [
@@ -167,76 +131,198 @@ describe("VisionHandoffService.transcribeForProvider", () => {
],
},
];
- const result = await svc.transcribeForProvider(messages, "umans/glm-5.2");
+ const result = await svc.prepareForProvider(messages, "umans/glm-5.2", {
+ conversationId: "conv-1",
+ });
expect(result).toHaveLength(1);
const chunks = result[0]?.chunks;
expect(chunks).toHaveLength(2);
+ // Text chunk unchanged.
expect(chunks?.[0]).toEqual({ type: "text", text: "Describe this" });
- // The image chunk was replaced with a transcribed text chunk.
+ // Image chunk → placeholder text.
expect(chunks?.[1]?.type).toBe("text");
- expect((chunks?.[1] as { text: string }).text).toContain("Image analysis");
- expect((chunks?.[1] as { text: string }).text).toContain("img1");
+ const placeholder = (chunks?.[1] as { text: string }).text;
+ expect(placeholder).toContain("Image 1");
+ expect(placeholder).toContain("consult_vision");
});
- it("caches transcription per unique image URL within a call", async () => {
+ it("assigns sequential image IDs across multiple messages", async () => {
const deps = makeDeps();
const svc = createVisionHandoffService(deps);
const messages: ChatMessage[] = [
- {
- role: "user",
- chunks: [
- { type: "image", url: "data:image/png;base64,same" },
- { type: "image", url: "data:image/png;base64,same" },
- ],
- },
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,a" }] },
+ { role: "assistant", chunks: [{ type: "text", text: "ok" }] },
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,b" }] },
];
- const result = await svc.transcribeForProvider(messages, "umans/glm-5.2");
- const chunks = result[0]?.chunks;
- // Both image chunks → text, same description (cached).
- expect(chunks).toHaveLength(2);
- expect((chunks?.[0] as { text: string }).text).toBe((chunks?.[1] as { text: string }).text);
- // The vision provider was called only once (cache hit on the second).
- const provider = deps.resolveModel("umans/kimi-k2.7")?.provider;
- expect((provider?.stream as ReturnType<typeof vi.fn>).mock.calls).toHaveLength(1);
+ const result = await svc.prepareForProvider(messages, "umans/glm-5.2", {
+ conversationId: "conv-1",
+ });
+ // First image → Image 1, second → Image 2.
+ expect((result[0]?.chunks[0] as { text: string }).text).toContain("Image 1");
+ // Assistant message unchanged.
+ expect(result[1]?.chunks[0]?.type).toBe("text");
+ expect((result[2]?.chunks[0] as { text: string }).text).toContain("Image 2");
});
- it("transcribes images in history messages too (non-vision model)", async () => {
+ it("registers images so getRegisteredImage can look them up", async () => {
const deps = makeDeps();
const svc = createVisionHandoffService(deps);
const messages: ChatMessage[] = [
- { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,hist" }] },
- { role: "assistant", chunks: [{ type: "text", text: "got it" }] },
- { role: "user", chunks: [{ type: "text", text: "and now?" }] },
+ {
+ role: "user",
+ chunks: [{ type: "image", url: "data:image/png;base64,registered" }],
+ },
];
- const result = await svc.transcribeForProvider(messages, "umans/glm-5.2");
- // First message's image chunk is now text.
- expect(result[0]?.chunks[0]?.type).toBe("text");
- expect((result[0]?.chunks[0] as { text: string }).text).toContain("Image analysis");
- // Assistant message unchanged.
- expect(result[1]?.chunks[0]?.type).toBe("text");
- // Last user message unchanged.
- expect(result[2]?.chunks[0]).toEqual({ type: "text", text: "and now?" });
+ await svc.prepareForProvider(messages, "umans/glm-5.2", { conversationId: "conv-42" });
+ const img = svc.getRegisteredImage("conv-42", 1);
+ expect(img?.url).toBe("data:image/png;base64,registered");
});
- it("uses a placeholder when no vision model is available (non-vision model)", async () => {
+ it("uses no-vision placeholder when no vision model is available", async () => {
const deps = makeDeps();
(deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]);
const svc = createVisionHandoffService(deps);
const messages: ChatMessage[] = [
{ role: "user", chunks: [{ type: "image", url: "data:image/png;base64,abc" }] },
];
- const result = await svc.transcribeForProvider(messages, "umans/glm-5.2");
- expect((result[0]?.chunks[0] as { text: string }).text).toContain("no vision-capable model");
+ const result = await svc.prepareForProvider(messages, "umans/glm-5.2", {
+ conversationId: "conv-1",
+ });
+ const text = (result[0]?.chunks[0] as { text: string }).text;
+ expect(text).toContain("no vision-capable model");
+ expect(text).not.toContain("consult_vision");
});
});
-describe("VisionHandoffService.readImageFile", () => {
- it("reads the file and transcribes it", async () => {
+describe("VisionHandoffService.consultVision", () => {
+ function makeOrchestratorDouble(response: string): {
+ orchestrator: NonNullable<
+ VisionHandoffDeps["resolveOrchestrator"] extends () => infer T ? T : never
+ >;
+ handleMessage: ReturnType<typeof vi.fn>;
+ } {
+ const handleMessage = vi.fn(
+ async (input: {
+ conversationId: string;
+ text: string;
+ onEvent: (event: AgentEvent) => void;
+ }): Promise<void> => {
+ input.onEvent({
+ type: "text-delta",
+ conversationId: input.conversationId,
+ turnId: "t1",
+ delta: response,
+ });
+ input.onEvent({
+ type: "done",
+ conversationId: input.conversationId,
+ turnId: "t1",
+ reason: "stop",
+ });
+ },
+ );
+ return { orchestrator: { handleMessage }, handleMessage };
+ }
+
+ it("opens a new consultation with a pasted image and returns convId + response", async () => {
const deps = makeDeps();
+ const { orchestrator, handleMessage } = makeOrchestratorDouble("The error is on line 12.");
+ deps.resolveOrchestrator = () => orchestrator;
const svc = createVisionHandoffService(deps);
- const result = await svc.readImageFile("screenshot.png", "/work");
- expect(deps.readFileAsDataUrl).toHaveBeenCalledWith("screenshot.png", "/work");
- expect(result).toContain("Image analysis");
- expect(result).toContain("FILE(screenshot.png)");
+
+ // Register an image first (as prepareForProvider would).
+ const messages: ChatMessage[] = [
+ { role: "user", chunks: [{ type: "image", url: "data:image/png;base64,img1" }] },
+ ];
+ await svc.prepareForProvider(messages, "umans/glm-5.2", { conversationId: "conv-1" });
+
+ const result = await svc.consultVision("What error is shown?", {
+ conversationId: "conv-1",
+ imageIds: [1],
+ });
+
+ expect("error" in result).toBe(false);
+ if (!("error" in result)) {
+ expect(result.conversationId).toBeTruthy();
+ expect(result.response).toContain("line 12");
+ expect(result.response).toContain(result.conversationId);
+ expect(result.response).toContain("dispatch CLI");
+ }
+ // The orchestrator was called with the vision model + the image.
+ expect(handleMessage).toHaveBeenCalledOnce();
+ const call = handleMessage.mock.calls[0]?.[0];
+ expect(call.modelName).toBe("umans/kimi-k2.7");
+ expect(call.images).toHaveLength(1);
+ expect(call.images?.[0]?.url).toBe("data:image/png;base64,img1");
+ });
+
+ it("opens a consultation with a file path image", async () => {
+ const deps = makeDeps();
+ const { orchestrator } = makeOrchestratorDouble("It's a diagram.");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+
+ const result = await svc.consultVision("What is this diagram?", {
+ conversationId: "conv-1",
+ path: "diagram.png",
+ cwd: "/work",
+ });
+
+ expect("error" in result).toBe(false);
+ expect(deps.readFileAsDataUrl).toHaveBeenCalledWith("diagram.png", "/work");
+ });
+
+ it("returns an error when imageId is not registered", async () => {
+ const deps = makeDeps();
+ const { orchestrator } = makeOrchestratorDouble("response");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ imageIds: [99], // not registered
+ });
+ expect("error" in result).toBe(true);
+ if ("error" in result) {
+ expect(result.error).toContain("Image 99");
+ }
+ });
+
+ it("returns an error when no orchestrator is available", async () => {
+ const deps = makeDeps();
+ // No resolveOrchestrator provided.
+ const svc = createVisionHandoffService(deps);
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ imageIds: [1],
+ });
+ expect("error" in result).toBe(true);
+ });
+
+ it("returns an error when no vision model is available", async () => {
+ const deps = makeDeps();
+ (deps.credentialStore.listCatalog as ReturnType<typeof vi.fn>).mockResolvedValue([]);
+ const { orchestrator } = makeOrchestratorDouble("response");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ imageIds: [1],
+ });
+ expect("error" in result).toBe(true);
+ if ("error" in result) {
+ expect(result.error).toContain("No vision-capable model");
+ }
+ });
+
+ it("returns an error when no image source is provided", async () => {
+ const deps = makeDeps();
+ const { orchestrator } = makeOrchestratorDouble("response");
+ deps.resolveOrchestrator = () => orchestrator;
+ const svc = createVisionHandoffService(deps);
+ const result = await svc.consultVision("What?", {
+ conversationId: "conv-1",
+ });
+ expect("error" in result).toBe(true);
});
});
diff --git a/packages/vision-handoff/src/service.ts b/packages/vision-handoff/src/service.ts
index 3f8462a..78f241f 100644
--- a/packages/vision-handoff/src/service.ts
+++ b/packages/vision-handoff/src/service.ts
@@ -3,40 +3,66 @@
* provider-agnostic vision handoff.
*
* Two capabilities:
- * 1. **Transcription for non-vision models** (`transcribeForProvider`): when a
- * user message carries images but the active model cannot see them, this
- * calls a vision-capable model (resolved from the catalog — any provider) to
- * describe each image, then replaces the image chunks with text. Universal:
- * it uses the standard `ProviderContract.stream` interface, never a
- * provider-specific vision endpoint.
- * 2. **`read_image` tool** (`readImageFile`): reads an image FILE from disk and
- * transcribes it via a vision-capable model, returning the text description
- * — so any model (vision or not) can analyze an image referenced in code.
+ * 1. **prepareForProvider** (`prepareForProvider`): when a user message carries
+ * images but the active model cannot see them, this replaces each image chunk
+ * with a numbered placeholder (telling the model to call `consult_vision`)
+ * and registers the image data in a per-conversation registry for tool
+ * access. Vision-capable models pass through unchanged (images flow natively).
+ * 2. **consult_vision tool** (`consultVision`): opens a NEW conversation tab with
+ * a vision-capable model (resolved from the catalog — any provider), attaches
+ * the image(s) + the model's specific question, waits for the response, and
+ * returns the conversation ID + the vision model's answer. The model (e.g.
+ * GLM 5.2) directs the analysis — asking exactly what it needs — instead of
+ * receiving a pre-emptive generic dump. Follow-up questions go through the
+ * dispatch CLI (the conversation ID is the bridge), not another tool call.
*
- * Effects (credential store, provider streaming, filesystem, fetch) are
- * injected. The pure decisions live in `pure.ts`. This shell wires them.
+ * Effects (credential store, orchestrator, filesystem) are injected. The pure
+ * decisions live in `pure.ts`. This shell wires them.
*/
import type { CredentialStore } from "@dispatch/credential-store";
import type {
+ AgentEvent,
ChatMessage,
Chunk,
+ ImageInput,
Logger,
ModelInfo,
ProviderContract,
- ProviderStreamOptions,
} from "@dispatch/kernel";
import { defineService, type ServiceHandle } from "@dispatch/kernel";
import {
- buildTranscriptionPrompt,
collectTextFromStream,
findVisionModelName,
+ formatConsultResult,
+ formatImagePlaceholder,
formatNoVisionPlaceholder,
- formatTranscriptionText,
isVisionCapable,
} from "./pure.js";
/**
+ * Minimal orchestrator interface the service needs to start vision consultation
+ * turns. Defined locally (not imported from session-orchestrator) to avoid a
+ * compile-time dependency — resolved lazily at runtime via a local handle keyed
+ * to the same service ID.
+ */
+export interface OrchestratorForVision {
+ readonly handleMessage: (input: {
+ readonly conversationId: string;
+ readonly text: string;
+ readonly onEvent: (event: AgentEvent) => void;
+ readonly modelName?: string;
+ readonly cwd?: string;
+ readonly images?: readonly ImageInput[];
+ readonly systemPrompt?: string;
+ }) => Promise<void>;
+}
+
+/** Local handle for the session-orchestrator service (same ID, no import dep). */
+export const orchestratorLocalHandle: ServiceHandle<OrchestratorForVision> =
+ defineService<OrchestratorForVision>("session-orchestrator/orchestrator");
+
+/**
* Resolved vision model — a provider + its model id, ready to stream from.
*/
export interface ResolvedVisionModel {
@@ -45,6 +71,12 @@ export interface ResolvedVisionModel {
readonly modelName: string;
}
+/** A registered image (looked up by the consult_vision tool via imageId). */
+interface RegisteredImage {
+ readonly url: string;
+ readonly mimeType?: string;
+}
+
/**
* Dependencies the service needs — all injected (no ambient state).
*/
@@ -56,24 +88,24 @@ export interface VisionHandoffDeps {
) => { provider: ProviderContract; model: string } | undefined;
/**
* Read a file from disk as a base64 data URL. Injected so the shell controls
- * the filesystem edge (and tests inject a fake). Returns the data URL, or
- * throws on error (the caller surfaces it as a tool error).
+ * the filesystem edge. Returns the data URL, or throws on error.
*/
readonly readFileAsDataUrl: (path: string, cwd?: string) => Promise<string>;
/**
- * Fetch an HTTP(S) URL to a data URL (for http image sources). Injected so
- * tests inject a fake. Optional — when absent, HTTP image URLs are passed to
- * the vision provider as-is (it fetches them).
+ * Lazily resolve the session-orchestrator (for starting vision consultation
+ * turns). Returns `undefined` when not available — `consult_vision` degrades
+ * with an error. Lazy so activation order doesn't matter.
*/
- readonly fetchUrlAsDataUrl?: (url: string) => Promise<string>;
+ readonly resolveOrchestrator?: () => OrchestratorForVision | undefined;
+ /** Generate a new conversation ID for a consultation. Defaults to crypto.randomUUID. */
+ readonly generateId?: () => string;
readonly logger?: Logger;
}
export interface VisionHandoffService {
/**
* Whether a given model (by catalog name) is vision-capable. Uses the
- * credential store's ModelInfo + the name heuristic. Async because ModelInfo
- * may require a listModels round-trip (cached by the credential store).
+ * credential store's ModelInfo + the name heuristic.
*/
readonly isVisionCapable: (modelName: string | undefined) => Promise<boolean>;
@@ -84,43 +116,54 @@ export interface VisionHandoffService {
readonly resolveVisionModel: (excludeName?: string) => Promise<ResolvedVisionModel | undefined>;
/**
- * Transcribe a single image URL to a text description via a vision-capable
- * model. Returns the description, or a placeholder string when no vision
- * model is available (does NOT throw — callers want graceful degradation).
- */
- readonly transcribeImage: (
- imageUrl: string,
- userQuestion: string | undefined,
- opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
- ) => Promise<string>;
-
- /**
* Transform a message list for the provider: if the active model is
* vision-capable, return messages unchanged (images pass through natively).
- * If NOT vision-capable, replace every `image` chunk with a text
- * description (transcribed via a vision model — once per unique image URL,
- * cached within the call) so a text-only model can still reason about the
- * images. Never throws — on failure an image becomes a placeholder note.
- *
- * The PERSISTED history is NOT modified by this (the caller persists the
- * original messages with images); this only transforms what the provider sees.
+ * If NOT vision-capable, replace every `image` chunk with a numbered
+ * placeholder (telling the model to call `consult_vision`) and register the
+ * image data in the per-conversation registry for tool access. The PERSISTED
+ * history is NOT modified — only what the provider sees. Never throws.
*/
- readonly transcribeForProvider: (
+ readonly prepareForProvider: (
messages: readonly ChatMessage[],
currentModelName: string | undefined,
- opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
+ opts?: {
+ readonly conversationId?: string;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
) => Promise<readonly ChatMessage[]>;
/**
- * Read an image FILE from disk and transcribe it (the `read_image` tool's
- * core). Returns the description text. Throws on filesystem error (the tool
- * surfaces it as a tool-error result).
+ * Look up a registered image by conversation ID + image ID. Returns
+ * `undefined` when the image isn't registered (e.g. after a server restart).
+ */
+ readonly getRegisteredImage: (
+ conversationId: string,
+ imageId: number,
+ ) => RegisteredImage | undefined;
+
+ /**
+ * Open a NEW vision consultation conversation: attach image(s) + the model's
+ * question to a vision-capable model, wait for the response, and return the
+ * conversation ID + the vision model's answer. The model drives the analysis
+ * — it asks exactly what it needs. Follow-ups go through the dispatch CLI.
+ *
+ * @returns The conversation ID + the vision model's response text, or an
+ * error string (never throws — the tool surfaces it).
*/
- readonly readImageFile: (
- path: string,
- cwd: string | undefined,
- opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
- ) => Promise<string>;
+ readonly consultVision: (
+ question: string,
+ opts: {
+ readonly conversationId: string;
+ readonly imageIds?: readonly number[];
+ readonly path?: string;
+ readonly cwd?: string;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
+ ) => Promise<
+ { readonly conversationId: string; readonly response: string } | { readonly error: string }
+ >;
}
export const visionHandoffHandle: ServiceHandle<VisionHandoffService> =
@@ -133,6 +176,12 @@ function hasImageChunks(messages: readonly ChatMessage[]): boolean {
export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHandoffService {
const log = deps.logger;
+ const generateId = deps.generateId ?? (() => crypto.randomUUID());
+
+ // Per-conversation image registry: conversationId → (imageId → image data).
+ // Populated by prepareForProvider; consulted by the consult_vision tool.
+ // In-memory only (cleared on restart — the user re-pastes if needed).
+ const imageRegistry = new Map<string, Map<number, RegisteredImage>>();
async function getInfo(modelName: string): Promise<ModelInfo | undefined> {
return deps.credentialStore.getModelInfo(modelName);
@@ -149,41 +198,6 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando
return { provider: resolved.provider, model: resolved.model, modelName: name };
}
- async function streamVisionText(
- vision: ResolvedVisionModel,
- imageUrl: string,
- prompt: string,
- opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
- ): Promise<string> {
- // Build a single-turn user message: [text prompt, image]. The vision model
- // receives the image natively via the OpenAI-compatible content array
- // (convertMessages serializes the image chunk to image_url).
- const userMessage: ChatMessage = {
- role: "user",
- chunks: [
- { type: "text", text: prompt },
- { type: "image", url: imageUrl },
- ],
- };
- const providerOpts: ProviderStreamOptions = {
- model: vision.model,
- // NOTE: temperature is deliberately OMITTED. Different vision providers
- // have different constraints (e.g. Moonshot/Kimi only allows temperature:
- // 1; others allow 0–2). Hardcoding any value risks an HTTP 400 from a
- // provider that rejects it. Omitting lets each provider use its own
- // default — the truly universal, provider-agnostic choice.
- // A short system prompt keeps the vision model focused on describing.
- systemPrompt:
- "You are a vision assistant. Describe images faithfully and thoroughly for a developer who cannot see them.",
- };
- const streamOpts: Parameters<ProviderContract["stream"]>[2] = {
- ...providerOpts,
- ...(opts?.logger !== undefined ? { logger: opts.logger } : {}),
- };
- const stream = vision.provider.stream([userMessage], [], streamOpts);
- return collectTextFromStream(stream);
- }
-
const service: VisionHandoffService = {
async isVisionCapable(modelName: string | undefined): Promise<boolean> {
if (modelName === undefined) return false;
@@ -193,35 +207,14 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando
resolveVisionModel,
- async transcribeImage(
- imageUrl: string,
- userQuestion: string | undefined,
- opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
- ): Promise<string> {
- const vision = await resolveVisionModel();
- if (vision === undefined) {
- log?.warn("vision-handoff: no vision-capable model available for transcription");
- return formatNoVisionPlaceholder();
- }
- const prompt = buildTranscriptionPrompt(userQuestion);
- try {
- const description = await streamVisionText(vision, imageUrl, prompt, opts);
- const trimmed = description.trim();
- if (trimmed.length === 0) {
- return "[Image analysis produced no output.]";
- }
- return formatTranscriptionText(trimmed, vision.modelName);
- } catch (err) {
- const msg = err instanceof Error ? err.message : String(err);
- log?.warn("vision-handoff: transcription failed", { error: msg });
- return `[Image analysis failed: ${msg}]`;
- }
- },
-
- async transcribeForProvider(
+ async prepareForProvider(
messages: readonly ChatMessage[],
currentModelName: string | undefined,
- opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
+ opts?: {
+ readonly conversationId?: string;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
): Promise<readonly ChatMessage[]> {
// Fast path: no images anywhere → nothing to do.
if (!hasImageChunks(messages)) return messages;
@@ -232,35 +225,41 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando
if (capable) return messages;
}
- // Non-vision model: transcribe each unique image URL once (cached).
- const cache = new Map<string, string>();
- const userText = messages
- .filter((m) => m.role === "user")
- .flatMap((m) => m.chunks)
- .filter((c): c is { type: "text"; text: string } => c.type === "text")
- .map((c) => c.text)
- .join(" ");
-
- async function transcribeCached(url: string): Promise<string> {
- const cached = cache.get(url);
- if (cached !== undefined) return cached;
- const description = await service.transcribeImage(url, userText, opts);
- cache.set(url, description);
- return description;
- }
+ // Non-vision model: check if a vision model is available at all.
+ const vision = await resolveVisionModel();
+ const convId = opts?.conversationId;
+ const placeholderFn =
+ vision !== undefined && convId !== undefined
+ ? (id: number) => formatImagePlaceholder(id)
+ : () => formatNoVisionPlaceholder();
+
+ // Replace each image chunk with a numbered placeholder. Assign sequential
+ // 1-based IDs across all messages and register each image in the
+ // per-conversation registry so the consult_vision tool can look it up.
+ let seqId = 0;
const result: ChatMessage[] = [];
for (const msg of messages) {
if (!msg.chunks.some((c) => c.type === "image")) {
result.push(msg);
continue;
}
- // Replace image chunks with transcribed text chunks; keep all else.
const newChunks: Chunk[] = [];
for (const chunk of msg.chunks) {
if (chunk.type === "image") {
- const description = await transcribeCached(chunk.url);
- newChunks.push({ type: "text", text: description });
+ seqId++;
+ if (convId !== undefined && vision !== undefined) {
+ let convImages = imageRegistry.get(convId);
+ if (convImages === undefined) {
+ convImages = new Map();
+ imageRegistry.set(convId, convImages);
+ }
+ convImages.set(seqId, {
+ url: chunk.url,
+ ...(chunk.mimeType !== undefined ? { mimeType: chunk.mimeType } : {}),
+ });
+ }
+ newChunks.push({ type: "text", text: placeholderFn(seqId) });
} else {
newChunks.push(chunk);
}
@@ -270,13 +269,109 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando
return result;
},
- async readImageFile(
- path: string,
- cwd: string | undefined,
- opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
- ): Promise<string> {
- const dataUrl = await deps.readFileAsDataUrl(path, cwd);
- return service.transcribeImage(dataUrl, undefined, opts);
+ getRegisteredImage(conversationId: string, imageId: number): RegisteredImage | undefined {
+ return imageRegistry.get(conversationId)?.get(imageId);
+ },
+
+ async consultVision(
+ question: string,
+ opts: {
+ readonly conversationId: string;
+ readonly imageIds?: readonly number[];
+ readonly path?: string;
+ readonly cwd?: string;
+ readonly signal?: AbortSignal;
+ readonly logger?: Logger;
+ },
+ ): Promise<
+ { readonly conversationId: string; readonly response: string } | { readonly error: string }
+ > {
+ const orchestrator = deps.resolveOrchestrator?.();
+ if (orchestrator === undefined) {
+ return {
+ error: "The session orchestrator is not available — cannot start a vision consultation.",
+ };
+ }
+
+ const vision = await resolveVisionModel();
+ if (vision === undefined) {
+ return {
+ error:
+ "No vision-capable model is available in the catalog. Install or configure one (e.g. kimi) to enable image analysis.",
+ };
+ }
+
+ // Collect image data URLs to attach.
+ const images: ImageInput[] = [];
+ if (opts.imageIds !== undefined) {
+ for (const id of opts.imageIds) {
+ const img = service.getRegisteredImage(opts.conversationId, id);
+ if (img === undefined) {
+ return {
+ error: `Image ${id} is not registered. It may have been lost after a server restart — ask the user to re-paste the image.`,
+ };
+ }
+ images.push({
+ url: img.url,
+ ...(img.mimeType !== undefined ? { mimeType: img.mimeType } : {}),
+ });
+ }
+ }
+ if (opts.path !== undefined) {
+ try {
+ const dataUrl = await deps.readFileAsDataUrl(opts.path, opts.cwd);
+ images.push({ url: dataUrl });
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ return { error: `Failed to read image file "${opts.path}": ${msg}` };
+ }
+ }
+ if (images.length === 0) {
+ return {
+ error:
+ "No image to consult about. Provide imageIds (for pasted images) or path (for a file).",
+ };
+ }
+
+ // Start a NEW conversation with the vision model.
+ const consultationId = generateId();
+ log?.info("vision-handoff: starting consultation", {
+ consultationId,
+ visionModel: vision.modelName,
+ imageCount: images.length,
+ fromConversation: opts.conversationId,
+ });
+
+ let responseText = "";
+ let errorMessage = "";
+ try {
+ await orchestrator.handleMessage({
+ conversationId: consultationId,
+ text: question,
+ images,
+ modelName: vision.modelName,
+ ...(opts.cwd !== undefined ? { cwd: opts.cwd } : {}),
+ systemPrompt:
+ "You are a vision assistant. A developer who cannot see images is asking you specific questions about an image they attached. Answer their question precisely and thoroughly.",
+ onEvent: (event: AgentEvent) => {
+ if (event.type === "text-delta") {
+ responseText += event.delta;
+ } else if (event.type === "error") {
+ errorMessage = event.message;
+ }
+ },
+ });
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ return { error: `Vision consultation failed: ${msg}` };
+ }
+
+ if (errorMessage.length > 0 && responseText.trim().length === 0) {
+ return { error: `Vision consultation failed: ${errorMessage}` };
+ }
+
+ const response = formatConsultResult(consultationId, responseText);
+ return { conversationId: consultationId, response };
},
};
diff --git a/packages/vision-handoff/src/tool.ts b/packages/vision-handoff/src/tool.ts
index 3995598..86be2ed 100644
--- a/packages/vision-handoff/src/tool.ts
+++ b/packages/vision-handoff/src/tool.ts
@@ -1,65 +1,134 @@
/**
- * read_image tool — lets any model (vision-capable or not) analyze an image
- * FILE on disk by handing it off to a vision-capable model.
+ * consult_vision tool — lets any model (vision-capable or not) consult a
+ * vision-capable model about an image by opening a NEW conversation tab.
*
- * The tool reads the image file into a base64 data URL, then asks the vision
- * handoff service to transcribe it (via a vision-capable model resolved from
- * the catalog) and returns the textual description as the tool result. This is
- * the universal mechanism: it works regardless of whether the active model has
- * vision, because the result is plain text the model reasons about.
+ * The tool attaches image(s) + the model's specific question to a vision-capable
+ * model (resolved from the catalog — e.g. Kimi), waits for the response, and
+ * returns the conversation ID + the vision model's answer. The MODEL directs the
+ * analysis — it asks exactly what it needs to know — instead of receiving a
+ * pre-emptive generic dump.
*
- * For images PASTED into the chat, the orchestrator's auto-transcription handles
- * them (no tool call needed). This tool is for images REFERENCED IN CODE by path
- * (e.g. a screenshot, diagram, or mockup the model discovered while reading files).
+ * For images PASTED into the chat, the model references them by `imageIds` (from
+ * the "[Image N attached]" placeholders the orchestrator injected). For image
+ * FILES on disk, the model passes a `path`.
+ *
+ * Follow-up questions are NOT handled by this tool — the model uses the dispatch
+ * CLI to continue the vision conversation (the returned conversation ID is the
+ * bridge; the model can load the `dispatch-cli` skill for the exact commands).
*/
import type { ToolContract, ToolExecuteContext, ToolResult } from "@dispatch/kernel";
import type { VisionHandoffService } from "./service.js";
-export function createReadImageTool(service: VisionHandoffService): ToolContract {
+export function createConsultVisionTool(service: VisionHandoffService): ToolContract {
return {
- name: "read_image",
+ name: "consult_vision",
description:
- "Read and analyze an image file on disk (PNG, JPEG, WebP, GIF). Returns a " +
- "detailed textual description of the image's contents — useful when you " +
- "encounter a screenshot, diagram, UI mockup, or chart referenced in the " +
- "codebase and need to understand what it shows. The analysis is performed " +
- "by a vision-capable model, so you can use this even if you cannot " +
- "directly view images. Pass a file path (relative to the cwd or absolute).",
+ "Consult a vision-capable model (e.g. Kimi) about an image by opening a new " +
+ "conversation tab. Attaches the image(s) + your specific question, waits for " +
+ "the vision model's response, and returns the conversation ID + the answer. " +
+ "Use this when you cannot view an image (e.g. a pasted screenshot or diagram) " +
+ "and need to know what it shows — ask a SPECIFIC question (e.g. 'What error " +
+ "message is on line 12?' rather than 'describe this image'). The conversation " +
+ "ID is returned so follow-up questions can be asked via the dispatch CLI.",
parameters: {
type: "object",
properties: {
+ question: {
+ type: "string",
+ description:
+ "Your specific question about the image. Be precise — the vision model " +
+ "will answer exactly this. E.g. 'What error message is displayed?' or " +
+ "'Compare the layout of these two screenshots.'",
+ },
+ imageIds: {
+ type: "array",
+ items: { type: "number" },
+ description:
+ "The IDs of pasted images to attach (from the '[Image N attached]' " +
+ "placeholders in the conversation). Pass multiple to attach several " +
+ "images to one consultation (e.g. [1, 2] to compare them).",
+ },
path: {
type: "string",
description:
- "Path to the image file to analyze. Relative paths resolve against " +
- "the conversation's working directory; absolute paths are used as-is.",
+ "Path to an image FILE on disk to attach (alternative to imageIds for " +
+ "code-referenced images). Relative paths resolve against the cwd.",
},
},
- required: ["path"],
+ required: ["question"],
},
concurrencySafe: true,
async execute(args: unknown, ctx: ToolExecuteContext): Promise<ToolResult> {
- const input = args as { path?: unknown } | null;
+ const input = args as {
+ question?: unknown;
+ imageIds?: unknown;
+ path?: unknown;
+ } | null;
+
+ const question = input?.question;
+ if (typeof question !== "string" || question.trim().length === 0) {
+ return {
+ content: "Error: 'question' is required and must be a non-empty string.",
+ isError: true,
+ };
+ }
+
+ const imageIds = input?.imageIds;
const path = input?.path;
- if (typeof path !== "string" || path.trim().length === 0) {
+
+ // Parse imageIds (must be an array of numbers if present).
+ let parsedImageIds: number[] | undefined;
+ if (imageIds !== undefined) {
+ if (!Array.isArray(imageIds)) {
+ return { content: "Error: 'imageIds' must be an array of numbers.", isError: true };
+ }
+ parsedImageIds = imageIds.filter((n): n is number => typeof n === "number");
+ if (parsedImageIds.length === 0) {
+ return { content: "Error: 'imageIds' must contain at least one number.", isError: true };
+ }
+ }
+
+ // path must be a string if present.
+ let parsedPath: string | undefined;
+ if (path !== undefined) {
+ if (typeof path !== "string" || path.trim().length === 0) {
+ return { content: "Error: 'path' must be a non-empty string.", isError: true };
+ }
+ parsedPath = path;
+ }
+
+ // At least one image source is required.
+ if (parsedImageIds === undefined && parsedPath === undefined) {
return {
- content: "Error: 'path' is required and must be a non-empty string.",
+ content:
+ "Error: provide 'imageIds' (for pasted images) or 'path' (for a file) " +
+ "to attach an image to the consultation.",
isError: true,
};
}
- const span = ctx.log.span("read_image.execute", { path });
+
+ const span = ctx.log.span("consult_vision.execute", {
+ imageCount: (parsedImageIds?.length ?? 0) + (parsedPath !== undefined ? 1 : 0),
+ });
try {
- const description = await service.readImageFile(path, ctx.cwd, {
+ const result = await service.consultVision(question, {
+ conversationId: ctx.conversationId ?? "",
+ ...(parsedImageIds !== undefined ? { imageIds: parsedImageIds } : {}),
+ ...(parsedPath !== undefined ? { path: parsedPath } : {}),
+ ...(ctx.cwd !== undefined ? { cwd: ctx.cwd } : {}),
signal: ctx.signal,
logger: ctx.log,
});
- span.end({ attrs: { descriptionLength: description.length } });
- return { content: description };
+ span.end({ attrs: { ok: !("error" in result) } });
+ if ("error" in result) {
+ return { content: result.error, isError: true };
+ }
+ return { content: result.response };
} catch (err: unknown) {
span.end({ err });
return {
- content: `Error reading image: ${err instanceof Error ? err.message : String(err)}`,
+ content: `Error during vision consultation: ${err instanceof Error ? err.message : String(err)}`,
isError: true,
};
}