feat(vision-handoff): implement vision for capable models and universal vision handoff

2 files changed, 66 insertions, 0 deletions

diff --git a/packages/transport-contract/src/contract.types.test.ts b/packages/transport-contract/src/contract.types.test.ts
index 9d3d904..34ff544 100644
--- a/packages/transport-contract/src/contract.types.test.ts
+++ b/packages/transport-contract/src/contract.types.test.ts
@@ -20,6 +20,7 @@ import type {
   LspServerState,
   LspStatusResponse,
   McpStatusResponse,
+  ModelsResponse,
   SetConversationComputerRequest,
   SetCwdRequest,
   SetWorkspaceDefaultComputerRequest,
@@ -55,6 +56,18 @@ const _chatWithoutComputer: ChatRequest = {
   message: "hello",
 };
 
+// ─── ChatRequest.images (additive optional) ──────────────────────────────────
+
+const _chatWithImages: ChatRequest = {
+  message: "What's in this screenshot?",
+  images: [{ url: "data:image/png;base64,iVBORw0KGgo=", mimeType: "image/png" }],
+};
+
+const _chatWithHttpImage: ChatRequest = {
+  message: "analyze this",
+  images: [{ url: "https://example.com/diagram.png" }],
+};
+
 // ─── Computer list / single response ─────────────────────────────────────────
 
 const _computer: Computer = {
@@ -255,6 +268,35 @@ describe("transport-contract types compile and are exported", () => {
     expect(_chatWithComputer.computerId).toBe("prod-box");
   });
 
+  // ─── ChatRequest.images (additive optional) ──────────────────────────────
+
+  it("ChatRequest: images is additive optional (omittable)", () => {
+    expect(_chatWithoutComputer.images).toBeUndefined();
+  });
+
+  it("ChatRequest: carries images (data URL) when set", () => {
+    expect(_chatWithImages.images).toHaveLength(1);
+    expect(_chatWithImages.images?.[0]?.url).toContain("base64");
+    expect(_chatWithImages.images?.[0]?.mimeType).toBe("image/png");
+  });
+
+  it("ChatRequest: carries images (http URL, mimeType optional)", () => {
+    expect(_chatWithHttpImage.images?.[0]?.url).toBe("https://example.com/diagram.png");
+    expect(_chatWithHttpImage.images?.[0]?.mimeType).toBeUndefined();
+  });
+
+  it("ModelsResponse: ModelMetadata carries optional vision flag", () => {
+    const resp: ModelsResponse = {
+      models: ["umans/kimi-k2.7", "umans/glm-5.2"],
+      modelInfo: {
+        "umans/kimi-k2.7": { contextWindow: 200000, vision: true },
+        "umans/glm-5.2": { contextWindow: 128000 },
+      },
+    };
+    expect(resp.modelInfo?.["umans/kimi-k2.7"]?.vision).toBe(true);
+    expect(resp.modelInfo?.["umans/glm-5.2"]?.vision).toBeUndefined();
+  });
+
   // ─── Computers ───────────────────────────────────────────────────────────
 
   it("ComputerListResponse: carries entries with usage counts", () => {
diff --git a/packages/transport-contract/src/index.ts b/packages/transport-contract/src/index.ts
index 6a9a29f..0444f29 100644
--- a/packages/transport-contract/src/index.ts
+++ b/packages/transport-contract/src/index.ts
@@ -26,6 +26,7 @@ import type {
   ComputerEntry,
   ConversationMeta,
   ConversationStatus,
+  ImageInput,
   QueuedMessage,
   ReasoningEffort,
   StoredChunk,
@@ -41,6 +42,8 @@ export type {
   ComputerEntry,
   ConversationMeta,
   ConversationStatus,
+  ImageChunk,
+  ImageInput,
   QueuedMessage,
   ReasoningEffort,
   StepMetrics,
@@ -68,6 +71,19 @@ export interface ChatRequest {
   readonly message: string;
 
   /**
+   * Images attached to this turn (e.g. a user-pasted screenshot). Each entry's
+   * `url` is a base64 data URL (`data:image/…;base64,…`) or an `http(s)://`
+   * URL. The server converts these to `image` chunks on the persisted user
+   * message. For a VISION-capable model (e.g. kimi), the images are passed
+   * through to the provider natively. For a NON-vision model (e.g. glm-5.2),
+   * the server's vision handoff transcribes each image to a text description
+   * (via a vision-capable model) and feeds that text instead — so a text-only
+   * model can still reason about the image's contents. Optional — omit for a
+   * text-only turn (backward compatible).
+   */
+  readonly images?: readonly ImageInput[];
+
+  /**
    * The model to use, as a model name in `<credentialName>/<model>` form — one
    * of the exact strings returned by `GET /models`. Omit to use the server's
    * default credential + model.
@@ -124,6 +140,14 @@ export interface ModelsResponse {
 /** Per-model metadata returned alongside the model catalog. */
 export interface ModelMetadata {
   readonly contextWindow?: number;
+  /**
+   * Whether this model can natively accept image input (vision/multimodal).
+   * When `true`, image chunks in a user message are passed through to the
+   * provider. When `false`/absent, the server's vision handoff transcribes
+   * images to text before the model sees them. A client may use this to show a
+   * vision badge in the model picker. Optional — absent when unknown.
+   */
+  readonly vision?: boolean;
 }
 
 /**