Merge branch 'feature/vision-handoff' into dev

# Conflicts:
#	packages/session-orchestrator/src/extension.ts
#	packages/session-orchestrator/src/orchestrator.ts

5 files changed, 218 insertions, 6 deletions

diff --git a/packages/openai-stream/src/convert-messages.test.ts b/packages/openai-stream/src/convert-messages.test.ts
index 3520eb5..57c7d81 100644
--- a/packages/openai-stream/src/convert-messages.test.ts
+++ b/packages/openai-stream/src/convert-messages.test.ts
@@ -35,6 +35,100 @@ describe("convertMessages", () => {
     expect(result).toEqual([{ role: "user", content: "Hello, world!" }]);
   });
 
+  it("converts a user message with a text + image chunk to a multimodal content array", () => {
+    const messages: ChatMessage[] = [
+      {
+        role: "user",
+        chunks: [
+          { type: "text", text: "What is in this image?" },
+          { type: "image", url: "data:image/png;base64,iVBORw0KGgo=" },
+        ],
+      },
+    ];
+
+    const result = convertMessages(messages);
+    expect(result).toEqual([
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "What is in this image?" },
+          { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw0KGgo=" } },
+        ],
+      },
+    ]);
+  });
+
+  it("converts an image-only user message (no text) to a content array with just the image", () => {
+    const messages: ChatMessage[] = [
+      {
+        role: "user",
+        chunks: [{ type: "image", url: "https://example.com/cat.png" }],
+      },
+    ];
+
+    const result = convertMessages(messages);
+    expect(result).toEqual([
+      {
+        role: "user",
+        content: [{ type: "image_url", image_url: { url: "https://example.com/cat.png" } }],
+      },
+    ]);
+  });
+
+  it("converts a user message with multiple images interspersed with text", () => {
+    const messages: ChatMessage[] = [
+      {
+        role: "user",
+        chunks: [
+          { type: "text", text: "Compare these:" },
+          { type: "image", url: "data:image/png;base64,aaa" },
+          { type: "text", text: "and" },
+          { type: "image", url: "data:image/jpeg;base64,bbb" },
+        ],
+      },
+    ];
+
+    const result = convertMessages(messages);
+    expect(result).toHaveLength(1);
+    const content = result[0]?.content;
+    expect(Array.isArray(content)).toBe(true);
+    if (Array.isArray(content)) {
+      expect(content).toHaveLength(4);
+      expect(content[0]).toEqual({ type: "text", text: "Compare these:" });
+      expect(content[1]).toEqual({
+        type: "image_url",
+        image_url: { url: "data:image/png;base64,aaa" },
+      });
+      expect(content[2]).toEqual({ type: "text", text: "and" });
+      expect(content[3]).toEqual({
+        type: "image_url",
+        image_url: { url: "data:image/jpeg;base64,bbb" },
+      });
+    }
+  });
+
+  it("skips empty text parts in a multimodal message but keeps images", () => {
+    const messages: ChatMessage[] = [
+      {
+        role: "user",
+        chunks: [
+          { type: "text", text: "" },
+          { type: "image", url: "data:image/png;base64,x" },
+        ],
+      },
+    ];
+
+    const result = convertMessages(messages);
+    const content = result[0]?.content;
+    expect(Array.isArray(content)).toBe(true);
+    if (Array.isArray(content)) {
+      // Empty text part is dropped; only the image remains.
+      expect(content).toEqual([
+        { type: "image_url", image_url: { url: "data:image/png;base64,x" } },
+      ]);
+    }
+  });
+
   it("converts an assistant message with text only", () => {
     const messages: ChatMessage[] = [
       {
diff --git a/packages/openai-stream/src/convert-messages.ts b/packages/openai-stream/src/convert-messages.ts
index e830243..eba3575 100644
--- a/packages/openai-stream/src/convert-messages.ts
+++ b/packages/openai-stream/src/convert-messages.ts
@@ -1,8 +1,28 @@
 import type { ChatMessage, Chunk } from "@dispatch/kernel";
 
+/** A text part within a multimodal OpenAI content array. */
+export interface OpenAITextPart {
+  readonly type: "text";
+  readonly text: string;
+}
+
+/** An image part within a multimodal OpenAI content array (OpenAI vision format). */
+export interface OpenAIImagePart {
+  readonly type: "image_url";
+  readonly image_url: { readonly url: string };
+}
+
+/**
+ * A part of a multimodal message content array. When a message has mixed text
+ * and image chunks, the content is serialized as an array of these parts
+ * (OpenAI's vision format). Plain-text messages keep a string `content` for
+ * byte-stability with providers that only accept strings.
+ */
+export type OpenAIContentPart = OpenAITextPart | OpenAIImagePart;
+
 export interface OpenAIMessage {
   readonly role: "system" | "user" | "assistant" | "tool";
-  readonly content: string | null;
+  readonly content: string | null | readonly OpenAIContentPart[];
   readonly tool_calls?: readonly OpenAIToolCall[];
   readonly tool_call_id?: string;
 }
@@ -49,6 +69,29 @@ function convertSystemMessage(msg: ChatMessage): OpenAIMessage {
 }
 
 function convertUserMessage(msg: ChatMessage): OpenAIMessage {
+  // If the message has image chunks, serialize as a multimodal content array
+  // (OpenAI vision format): text parts + image_url parts in chunk order.
+  // Plain text-only messages keep a string `content` for byte-stability with
+  // providers that only accept a string (and to keep prompt-cache prefixes
+  // unchanged for the common no-image case).
+  const hasImage = msg.chunks.some((c) => c.type === "image");
+  if (hasImage) {
+    const parts: OpenAIContentPart[] = [];
+    for (const chunk of msg.chunks) {
+      if (chunk.type === "text") {
+        if (chunk.text.length > 0) {
+          parts.push({ type: "text", text: chunk.text });
+        }
+      } else if (chunk.type === "image") {
+        parts.push({ type: "image_url", image_url: { url: chunk.url } });
+      }
+      // Non-text/non-image chunks (tool-call, thinking, etc.) are not part of a
+      // user message's provider content and are skipped here.
+    }
+    // An image-only message (no text) still needs at least the image part.
+    return { role: "user", content: parts.length > 0 ? parts : "" };
+  }
+
   const text = msg.chunks
     .filter((c): c is Extract<Chunk, { type: "text" }> => c.type === "text")
     .map((c) => c.text)
diff --git a/packages/openai-stream/src/index.ts b/packages/openai-stream/src/index.ts
index bd2f673..3f76b99 100644
--- a/packages/openai-stream/src/index.ts
+++ b/packages/openai-stream/src/index.ts
@@ -1,8 +1,14 @@
-export type { OpenAIMessage, OpenAIToolCall } from "./convert-messages.js";
+export type {
+  OpenAIContentPart,
+  OpenAIImagePart,
+  OpenAIMessage,
+  OpenAITextPart,
+  OpenAIToolCall,
+} from "./convert-messages.js";
 export { convertMessages } from "./convert-messages.js";
 export type { OpenAITool } from "./convert-tools.js";
 export { convertTools } from "./convert-tools.js";
-export { parseModelList } from "./listModels.js";
+export { isVisionModelId, parseModelList } from "./listModels.js";
 export { parseSSELines } from "./parse-sse.js";
 export type { CreateOpenAICompatProviderOpts } from "./provider.js";
 export { createOpenAICompatProvider } from "./provider.js";
diff --git a/packages/openai-stream/src/listModels.test.ts b/packages/openai-stream/src/listModels.test.ts
index c2438bc..2e3b1a3 100644
--- a/packages/openai-stream/src/listModels.test.ts
+++ b/packages/openai-stream/src/listModels.test.ts
@@ -1,7 +1,7 @@
 import type { ApiKeyCredentials, ModelInfo, ProviderContract } from "@dispatch/kernel";
 import type { FetchLike } from "@dispatch/trace-replay";
 import { describe, expect, it, vi } from "vitest";
-import { parseModelList } from "./listModels.js";
+import { isVisionModelId, parseModelList } from "./listModels.js";
 import { createOpenAICompatProvider } from "./provider.js";
 
 function makeProvider(fetchFn: FetchLike, apiKey = "sk-test-1234567890abcdef"): ProviderContract {
@@ -35,6 +35,53 @@ describe("listModels — pure mapping (parseModelList)", () => {
     const result = parseModelList([]);
     expect(result).toEqual([]);
   });
+
+  it("extracts contextWindow from common field names", () => {
+    const result = parseModelList([
+      { id: "m1", context_length: 128000 },
+      { id: "m2", context_window: 200000 },
+      { id: "m3", max_context_length: 64000 },
+      { id: "m4", max_tokens: 8000 },
+    ]);
+    expect(result).toEqual([
+      { id: "m1", contextWindow: 128000 },
+      { id: "m2", contextWindow: 200000 },
+      { id: "m3", contextWindow: 64000 },
+      { id: "m4", contextWindow: 8000 },
+    ]);
+  });
+});
+
+describe("listModels — vision capability detection", () => {
+  it("isVisionModelId returns true for umans kimi and qwen model ids", () => {
+    expect(isVisionModelId("umans-kimi-k2.7")).toBe(true);
+    expect(isVisionModelId("Umans-Kimi-K2.7")).toBe(true); // case-insensitive
+    expect(isVisionModelId("umans-qwen3.6-35b-a3b")).toBe(true);
+  });
+
+  it("isVisionModelId returns false for non-vision model ids", () => {
+    expect(isVisionModelId("umans-glm-5.2")).toBe(false);
+    expect(isVisionModelId("umans-coder")).toBe(false);
+    expect(isVisionModelId("umans-flash")).toBe(false);
+    expect(isVisionModelId("kimi-k2.7-code")).toBe(false); // opencode kimi, not umans
+    expect(isVisionModelId("qwen3.7-max")).toBe(false); // opencode qwen, not umans
+    expect(isVisionModelId("deepseek-v4-flash")).toBe(false);
+  });
+
+  it("parseModelList sets vision: true on umans kimi and qwen models only", () => {
+    const result = parseModelList([
+      { id: "umans-kimi-k2.7", context_length: 262144 },
+      { id: "umans-qwen3.6-35b-a3b", context_length: 262144 },
+      { id: "umans-glm-5.2", context_length: 405504 },
+      { id: "umans-coder" },
+    ]);
+    expect(result).toEqual([
+      { id: "umans-kimi-k2.7", contextWindow: 262144, vision: true },
+      { id: "umans-qwen3.6-35b-a3b", contextWindow: 262144, vision: true },
+      { id: "umans-glm-5.2", contextWindow: 405504 },
+      { id: "umans-coder" },
+    ]);
+  });
 });
 
 describe("listModels — provider contract", () => {
diff --git a/packages/openai-stream/src/listModels.ts b/packages/openai-stream/src/listModels.ts
index 0e94c43..df116b0 100644
--- a/packages/openai-stream/src/listModels.ts
+++ b/packages/openai-stream/src/listModels.ts
@@ -24,17 +24,39 @@ interface OpenAIModelListResponse {
 }
 
 /**
+ * Whether a model id is vision-capable (can natively accept image input).
+ *
+ * The OpenAI-compatible `/models` endpoint does not reliably report image
+ * capabilities, so this is a hardcoded heuristic by model id: the Umans Kimi
+ * (`umans-kimi-k2.7`) and Umans Qwen (`umans-qwen3.6-35b-a3b`) models are
+ * vision-capable; all others are treated as non-vision. This is the single
+ * source of truth — the orchestrator's vision handoff and the `consult_vision`
+ * tool both consult the `ModelInfo.vision` flag this sets, so adding a model
+ * here enables vision everywhere. Pure: id → boolean, no I/O.
+ *
+ * (When an endpoint gains reliable vision reporting, this can be replaced with
+ * a real capability check without changing callers.)
+ */
+export function isVisionModelId(id: string): boolean {
+  const lower = id.toLowerCase();
+  return lower.includes("umans-kimi") || lower.includes("umans-qwen");
+}
+
+/**
  * Pure mapping: raw OpenAI-compatible model list → ModelInfo[].
- * Extracts `contextWindow` from common field names (providers vary).
- * Extracted for direct unit testing with no I/O.
+ * Extracts `contextWindow` from common field names (providers vary) and
+ * detects vision capability via {@link isVisionModelId}. Extracted for direct
+ * unit testing with no I/O.
  */
 export function parseModelList(data: readonly OpenAIModelEntry[]): readonly ModelInfo[] {
   return data.map((entry) => {
     const contextWindow =
       entry.context_length ?? entry.context_window ?? entry.max_context_length ?? entry.max_tokens;
+    const vision = isVisionModelId(entry.id);
     return {
       id: entry.id,
       ...(contextWindow !== undefined ? { contextWindow } : {}),
+      ...(vision ? { vision } : {}),
     };
   });
 }