feat(vision-handoff): implement vision for capable models and universal vision handoff

3 files changed, 115 insertions, 5 deletions

diff --git a/packages/transport-http/src/app.ts b/packages/transport-http/src/app.ts
index 4fb295e..a9a23da 100644
--- a/packages/transport-http/src/app.ts
+++ b/packages/transport-http/src/app.ts
@@ -294,11 +294,14 @@ export function createApp(opts: CreateServerOptions): Hono {
   app.get("/models", async (c) => {
     try {
       const models = await opts.credentialStore.listCatalog();
-      const modelInfo: Record<string, { contextWindow?: number }> = {};
+      const modelInfo: Record<string, { contextWindow?: number; vision?: boolean }> = {};
       for (const modelName of models) {
         const info = await opts.credentialStore.getModelInfo(modelName);
-        if (info?.contextWindow !== undefined) {
-          modelInfo[modelName] = { contextWindow: info.contextWindow };
+        if (info?.contextWindow !== undefined || info?.vision === true) {
+          const entry: { contextWindow?: number; vision?: boolean } = {};
+          if (info?.contextWindow !== undefined) entry.contextWindow = info.contextWindow;
+          if (info?.vision === true) entry.vision = true;
+          modelInfo[modelName] = entry;
         }
       }
       const body: ModelsResponse = {
@@ -398,8 +401,16 @@ export function createApp(opts: CreateServerOptions): Hono {
       return c.json({ error: result.error }, 400);
     }
 
-    const { conversationId, message, model, cwd, computerId, reasoningEffort, workspaceId } =
-      result;
+    const {
+      conversationId,
+      message,
+      model,
+      cwd,
+      computerId,
+      reasoningEffort,
+      workspaceId,
+      images,
+    } = result;
     log.info("chat: request accepted", {
       conversationId,
       hasModel: model !== undefined,
@@ -407,6 +418,7 @@ export function createApp(opts: CreateServerOptions): Hono {
       hasComputerId: computerId !== undefined,
       hasReasoningEffort: reasoningEffort !== undefined,
       hasWorkspaceId: workspaceId !== undefined,
+      imageCount: images?.length ?? 0,
     });
 
     const events: AgentEvent[] = [];
@@ -457,6 +469,7 @@ export function createApp(opts: CreateServerOptions): Hono {
       ...(computerId !== undefined ? { computerId } : {}),
       ...(reasoningEffort !== undefined ? { reasoningEffort } : {}),
       ...(workspaceId !== undefined ? { workspaceId } : {}),
+      ...(images !== undefined ? { images } : {}),
     };
 
     opts.orchestrator
diff --git a/packages/transport-http/src/logic.test.ts b/packages/transport-http/src/logic.test.ts
index fc8302e..67632f3 100644
--- a/packages/transport-http/src/logic.test.ts
+++ b/packages/transport-http/src/logic.test.ts
@@ -182,6 +182,69 @@ describe("parseChatBody", () => {
       expect(result.reasoningEffort).toBeUndefined();
     }
   });
+
+  // ── images ──────────────────────────────────────────────────────────────
+
+  it("parses images array with data URLs", () => {
+    const result = parseChatBody(
+      {
+        message: "what is this?",
+        images: [
+          { url: "data:image/png;base64,aaa" },
+          { url: "data:image/jpeg;base64,bbb", mimeType: "image/jpeg" },
+        ],
+      },
+      fakeId,
+    );
+    expect(isParseError(result)).toBe(false);
+    if (!isParseError(result)) {
+      expect(result.images).toHaveLength(2);
+      expect(result.images?.[0]?.url).toBe("data:image/png;base64,aaa");
+      expect(result.images?.[1]?.mimeType).toBe("image/jpeg");
+    }
+  });
+
+  it("parses images with http URLs", () => {
+    const result = parseChatBody(
+      { message: "hi", images: [{ url: "https://example.com/x.png" }] },
+      fakeId,
+    );
+    expect(isParseError(result)).toBe(false);
+    if (!isParseError(result)) {
+      expect(result.images?.[0]?.url).toBe("https://example.com/x.png");
+    }
+  });
+
+  it("returns error when images is not an array", () => {
+    const result = parseChatBody({ message: "hi", images: "not-an-array" }, fakeId);
+    expect(isParseError(result)).toBe(true);
+  });
+
+  it("returns error when an image lacks a url", () => {
+    const result = parseChatBody({ message: "hi", images: [{ mimeType: "image/png" }] }, fakeId);
+    expect(isParseError(result)).toBe(true);
+  });
+
+  it("returns error when an image url is empty", () => {
+    const result = parseChatBody({ message: "hi", images: [{ url: "" }] }, fakeId);
+    expect(isParseError(result)).toBe(true);
+  });
+
+  it("omits images when absent (backward compatible)", () => {
+    const result = parseChatBody({ message: "hi" }, fakeId);
+    expect(isParseError(result)).toBe(false);
+    if (!isParseError(result)) {
+      expect(result.images).toBeUndefined();
+    }
+  });
+
+  it("omits images when the array is empty", () => {
+    const result = parseChatBody({ message: "hi", images: [] }, fakeId);
+    expect(isParseError(result)).toBe(false);
+    if (!isParseError(result)) {
+      expect(result.images).toBeUndefined();
+    }
+  });
 });
 
 describe("parseSinceSeq", () => {
diff --git a/packages/transport-http/src/logic.ts b/packages/transport-http/src/logic.ts
index 97ad426..a928147 100644
--- a/packages/transport-http/src/logic.ts
+++ b/packages/transport-http/src/logic.ts
@@ -55,6 +55,13 @@ export interface ChatCommand {
   readonly computerId?: string;
   readonly reasoningEffort?: ReasoningEffort;
   readonly workspaceId?: string;
+  /**
+   * Images attached to this turn (data URLs or http URLs). Parsed from the
+   * `ChatRequest.images` field; forwarded to the orchestrator which converts
+   * them to `image` chunks on the user message. Each entry must have a non-empty
+   * string `url`; `mimeType` is optional.
+   */
+  readonly images?: readonly { readonly url: string; readonly mimeType?: string }[];
 }
 
 export interface ParseError {
@@ -121,6 +128,33 @@ export function parseChatBody(body: unknown, generateId: () => string): ParseRes
     (result as { workspaceId?: string }).workspaceId = obj.workspaceId;
   }
 
+  if (obj.images !== undefined) {
+    if (!Array.isArray(obj.images)) {
+      return { error: "Field 'images' must be an array" };
+    }
+    const images: { url: string; mimeType?: string }[] = [];
+    for (const entry of obj.images) {
+      if (entry === null || typeof entry !== "object") {
+        return { error: "Each image must be an object with a 'url' string" };
+      }
+      const img = entry as { url?: unknown; mimeType?: unknown };
+      if (typeof img.url !== "string" || img.url.length === 0) {
+        return { error: "Each image must have a non-empty string 'url'" };
+      }
+      const parsed: { url: string; mimeType?: string } = { url: img.url };
+      if (img.mimeType !== undefined) {
+        if (typeof img.mimeType !== "string") {
+          return { error: "Field 'mimeType' on an image must be a string" };
+        }
+        parsed.mimeType = img.mimeType;
+      }
+      images.push(parsed);
+    }
+    if (images.length > 0) {
+      (result as { images?: readonly { url: string; mimeType?: string }[] }).images = images;
+    }
+  }
+
   return result;
 }