From d5633cf6e007eaf8255a44529a638d2466a74ba3 Mon Sep 17 00:00:00 2001
From: Adam Malczewski <github@tradam.dev>
Date: Sat, 27 Jun 2026 03:40:38 +0900
Subject: feat(vision-handoff): implement vision for capable models and
 universal vision handoff

---
 packages/wire/src/index.test.ts | 31 ++++++++++++++++++++++++++++-
 packages/wire/src/index.ts      | 43 ++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 72 insertions(+), 2 deletions(-)

(limited to 'packages/wire')

diff --git a/packages/wire/src/index.test.ts b/packages/wire/src/index.test.ts
index 3f07e00..81d10c1 100644
--- a/packages/wire/src/index.test.ts
+++ b/packages/wire/src/index.test.ts
@@ -8,7 +8,7 @@
  */
 
 import { describe, expect, it } from "vitest";
-import type { Computer, ComputerEntry, Workspace } from "./index.js";
+import type { Chunk, Computer, ComputerEntry, ImageChunk, ImageInput, Workspace } from "./index.js";
 
 describe("@dispatch/wire — Computer / Workspace shapes", () => {
   it("a Computer literal satisfies the Computer type", () => {
@@ -57,3 +57,32 @@ describe("@dispatch/wire — Computer / Workspace shapes", () => {
     expect(local.defaultComputerId).toBeNull();
   });
 });
+
+describe("@dispatch/wire — ImageChunk / ImageInput shapes", () => {
+  it("an ImageChunk carries a data URL and optional mimeType", () => {
+    const c: ImageChunk = {
+      type: "image",
+      url: "data:image/png;base64,iVBORw0KGgo=",
+      mimeType: "image/png",
+    };
+    expect(c.type).toBe("image");
+    expect(c.url).toContain("base64");
+    expect(c.mimeType).toBe("image/png");
+  });
+
+  it("an ImageChunk with only a url is valid (mimeType optional)", () => {
+    const c: ImageChunk = { type: "image", url: "https://example.com/cat.png" };
+    expect(c.mimeType).toBeUndefined();
+  });
+
+  it("ImageInput mirrors ImageChunk's url semantics", () => {
+    const input: ImageInput = { url: "data:image/jpeg;base64,/9j/4AAQ" };
+    expect(input.url).toContain("jpeg");
+  });
+
+  it("ImageChunk is a member of the Chunk union (assignable)", () => {
+    const chunk: Chunk = { type: "image", url: "data:image/png;base64,x" };
+    // Compile-time proof: an ImageChunk satisfies the Chunk union.
+    expect(chunk.type).toBe("image");
+  });
+});
diff --git a/packages/wire/src/index.ts b/packages/wire/src/index.ts
index 16b7023..d6ea1c1 100644
--- a/packages/wire/src/index.ts
+++ b/packages/wire/src/index.ts
@@ -36,7 +36,8 @@ export type Chunk =
   | ToolCallChunk
   | ToolResultChunk
   | ErrorChunk
-  | SystemChunk;
+  | SystemChunk
+  | ImageChunk;
 
 /** A piece of plain text content from the assistant or user. */
 export interface TextChunk {
@@ -112,6 +113,46 @@ export interface SystemChunk {
   readonly text: string;
 }
 
+/**
+ * An image attached to a message (e.g. a user-pasted screenshot or pasted
+ * photo). Carries a `url` that is EITHER a base64 data URL
+ * (`data:image/png;base64,…`) OR an `http(s)://` URL. Vision-capable models
+ * receive it natively (the provider serializes it to its image-content
+ * format); non-vision models never see it directly — the orchestrator's
+ * **vision handoff** transcribes it to a text description (via a
+ * vision-capable model) and feeds that text instead, so a text-only model can
+ * still reason about the image's contents.
+ *
+ * When a transcription was performed, it is persisted as a separate `text`
+ * chunk alongside the `image` chunk in the SAME user message, so the
+ * description is reused on every later turn (no re-transcription) and a
+ * client renders both the original image and its textual analysis.
+ */
+export interface ImageChunk {
+  readonly type: "image";
+  /** Image source: a base64 data URL (`data:image/…;base64,…`) or an `http(s)://` URL. */
+  readonly url: string;
+  /**
+   * Optional MIME type of the image (e.g. `"image/png"`). Inferred from the
+   * data URL when absent; present so a client can render an icon/label without
+   * parsing the URL. Optional — callers that only have a URL omit it.
+   */
+  readonly mimeType?: string;
+}
+
+/**
+ * An image a client attaches to a chat message (`ChatRequest.images`). The
+ * transport-facing input shape; the orchestrator converts each `ImageInput`
+ * into an `ImageChunk` on the persisted user message. Carries the same `url`
+ * semantics as `ImageChunk.url`.
+ */
+export interface ImageInput {
+  /** Image source: a base64 data URL (`data:image/…;base64,…`) or an `http(s)://` URL. */
+  readonly url: string;
+  /** Optional MIME type (e.g. `"image/png"`). Optional — inferred from the data URL when absent. */
+  readonly mimeType?: string;
+}
+
 /**
  * A chat message: a role plus an ordered sequence of chunks. Messages are the
  * unit passed to and from the provider; chunks are the unit persisted and
-- 
cgit v1.2.3