feat(vision-handoff): model-directed consult_vision tool replacing auto-transcription

1 files changed, 98 insertions, 29 deletions

diff --git a/packages/vision-handoff/src/tool.ts b/packages/vision-handoff/src/tool.ts
index 3995598..86be2ed 100644
--- a/packages/vision-handoff/src/tool.ts
+++ b/packages/vision-handoff/src/tool.ts
@@ -1,65 +1,134 @@
 /**
- * read_image tool — lets any model (vision-capable or not) analyze an image
- * FILE on disk by handing it off to a vision-capable model.
+ * consult_vision tool — lets any model (vision-capable or not) consult a
+ * vision-capable model about an image by opening a NEW conversation tab.
  *
- * The tool reads the image file into a base64 data URL, then asks the vision
- * handoff service to transcribe it (via a vision-capable model resolved from
- * the catalog) and returns the textual description as the tool result. This is
- * the universal mechanism: it works regardless of whether the active model has
- * vision, because the result is plain text the model reasons about.
+ * The tool attaches image(s) + the model's specific question to a vision-capable
+ * model (resolved from the catalog — e.g. Kimi), waits for the response, and
+ * returns the conversation ID + the vision model's answer. The MODEL directs the
+ * analysis — it asks exactly what it needs to know — instead of receiving a
+ * pre-emptive generic dump.
  *
- * For images PASTED into the chat, the orchestrator's auto-transcription handles
- * them (no tool call needed). This tool is for images REFERENCED IN CODE by path
- * (e.g. a screenshot, diagram, or mockup the model discovered while reading files).
+ * For images PASTED into the chat, the model references them by `imageIds` (from
+ * the "[Image N attached]" placeholders the orchestrator injected). For image
+ * FILES on disk, the model passes a `path`.
+ *
+ * Follow-up questions are NOT handled by this tool — the model uses the dispatch
+ * CLI to continue the vision conversation (the returned conversation ID is the
+ * bridge; the model can load the `dispatch-cli` skill for the exact commands).
  */
 
 import type { ToolContract, ToolExecuteContext, ToolResult } from "@dispatch/kernel";
 import type { VisionHandoffService } from "./service.js";
 
-export function createReadImageTool(service: VisionHandoffService): ToolContract {
+export function createConsultVisionTool(service: VisionHandoffService): ToolContract {
   return {
-    name: "read_image",
+    name: "consult_vision",
     description:
-      "Read and analyze an image file on disk (PNG, JPEG, WebP, GIF). Returns a " +
-      "detailed textual description of the image's contents — useful when you " +
-      "encounter a screenshot, diagram, UI mockup, or chart referenced in the " +
-      "codebase and need to understand what it shows. The analysis is performed " +
-      "by a vision-capable model, so you can use this even if you cannot " +
-      "directly view images. Pass a file path (relative to the cwd or absolute).",
+      "Consult a vision-capable model (e.g. Kimi) about an image by opening a new " +
+      "conversation tab. Attaches the image(s) + your specific question, waits for " +
+      "the vision model's response, and returns the conversation ID + the answer. " +
+      "Use this when you cannot view an image (e.g. a pasted screenshot or diagram) " +
+      "and need to know what it shows — ask a SPECIFIC question (e.g. 'What error " +
+      "message is on line 12?' rather than 'describe this image'). The conversation " +
+      "ID is returned so follow-up questions can be asked via the dispatch CLI.",
     parameters: {
       type: "object",
       properties: {
+        question: {
+          type: "string",
+          description:
+            "Your specific question about the image. Be precise — the vision model " +
+            "will answer exactly this. E.g. 'What error message is displayed?' or " +
+            "'Compare the layout of these two screenshots.'",
+        },
+        imageIds: {
+          type: "array",
+          items: { type: "number" },
+          description:
+            "The IDs of pasted images to attach (from the '[Image N attached]' " +
+            "placeholders in the conversation). Pass multiple to attach several " +
+            "images to one consultation (e.g. [1, 2] to compare them).",
+        },
         path: {
           type: "string",
           description:
-            "Path to the image file to analyze. Relative paths resolve against " +
-            "the conversation's working directory; absolute paths are used as-is.",
+            "Path to an image FILE on disk to attach (alternative to imageIds for " +
+            "code-referenced images). Relative paths resolve against the cwd.",
         },
       },
-      required: ["path"],
+      required: ["question"],
     },
     concurrencySafe: true,
     async execute(args: unknown, ctx: ToolExecuteContext): Promise<ToolResult> {
-      const input = args as { path?: unknown } | null;
+      const input = args as {
+        question?: unknown;
+        imageIds?: unknown;
+        path?: unknown;
+      } | null;
+
+      const question = input?.question;
+      if (typeof question !== "string" || question.trim().length === 0) {
+        return {
+          content: "Error: 'question' is required and must be a non-empty string.",
+          isError: true,
+        };
+      }
+
+      const imageIds = input?.imageIds;
       const path = input?.path;
-      if (typeof path !== "string" || path.trim().length === 0) {
+
+      // Parse imageIds (must be an array of numbers if present).
+      let parsedImageIds: number[] | undefined;
+      if (imageIds !== undefined) {
+        if (!Array.isArray(imageIds)) {
+          return { content: "Error: 'imageIds' must be an array of numbers.", isError: true };
+        }
+        parsedImageIds = imageIds.filter((n): n is number => typeof n === "number");
+        if (parsedImageIds.length === 0) {
+          return { content: "Error: 'imageIds' must contain at least one number.", isError: true };
+        }
+      }
+
+      // path must be a string if present.
+      let parsedPath: string | undefined;
+      if (path !== undefined) {
+        if (typeof path !== "string" || path.trim().length === 0) {
+          return { content: "Error: 'path' must be a non-empty string.", isError: true };
+        }
+        parsedPath = path;
+      }
+
+      // At least one image source is required.
+      if (parsedImageIds === undefined && parsedPath === undefined) {
         return {
-          content: "Error: 'path' is required and must be a non-empty string.",
+          content:
+            "Error: provide 'imageIds' (for pasted images) or 'path' (for a file) " +
+            "to attach an image to the consultation.",
           isError: true,
         };
       }
-      const span = ctx.log.span("read_image.execute", { path });
+
+      const span = ctx.log.span("consult_vision.execute", {
+        imageCount: (parsedImageIds?.length ?? 0) + (parsedPath !== undefined ? 1 : 0),
+      });
       try {
-        const description = await service.readImageFile(path, ctx.cwd, {
+        const result = await service.consultVision(question, {
+          conversationId: ctx.conversationId ?? "",
+          ...(parsedImageIds !== undefined ? { imageIds: parsedImageIds } : {}),
+          ...(parsedPath !== undefined ? { path: parsedPath } : {}),
+          ...(ctx.cwd !== undefined ? { cwd: ctx.cwd } : {}),
           signal: ctx.signal,
           logger: ctx.log,
         });
-        span.end({ attrs: { descriptionLength: description.length } });
-        return { content: description };
+        span.end({ attrs: { ok: !("error" in result) } });
+        if ("error" in result) {
+          return { content: result.error, isError: true };
+        }
+        return { content: result.response };
       } catch (err: unknown) {
         span.end({ err });
         return {
-          content: `Error reading image: ${err instanceof Error ? err.message : String(err)}`,
+          content: `Error during vision consultation: ${err instanceof Error ? err.message : String(err)}`,
           isError: true,
         };
       }