feat(vision-handoff): implement vision for capable models and universal vision handoff

4 files changed, 171 insertions, 5 deletions

diff --git a/packages/session-orchestrator/src/extension.ts b/packages/session-orchestrator/src/extension.ts
index 5afffd8..d080e90 100644
--- a/packages/session-orchestrator/src/extension.ts
+++ b/packages/session-orchestrator/src/extension.ts
@@ -11,6 +11,7 @@ import {
   createSessionOrchestrator,
   createWarmService,
   sessionOrchestratorHandle,
+  visionHandoffLocalHandle,
 } from "./orchestrator.js";
 import { selectFirstProvider } from "./pure.js";
 import { filterRemoteIncompatibleTools, toolsFilter } from "./tools-filter.js";
@@ -93,6 +94,20 @@ export function activate(host: HostAPI): void {
         return undefined;
       }
     },
+    resolveVisionHandoff: () => {
+      // Lazily resolve the vision-handoff service. Returns undefined when the
+      // vision-handoff extension isn't loaded (images pass through unchanged —
+      // correct for vision-capable models; the feature degrades off cleanly for
+      // text-only turns). Lazy so activation order doesn't matter; the
+      // activated-manifests guard avoids a getService throw when absent.
+      const loaded = host.getExtensions().some((m) => m.id === "vision-handoff");
+      if (!loaded) return undefined;
+      try {
+        return host.getService(visionHandoffLocalHandle);
+      } catch {
+        return undefined;
+      }
+    },
   });
 
   host.provideService(sessionOrchestratorHandle, orchestrator);
diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts
index 96cd3a3..ac1eaf4 100644
--- a/packages/session-orchestrator/src/orchestrator.ts
+++ b/packages/session-orchestrator/src/orchestrator.ts
@@ -5,6 +5,7 @@ import type {
   CompactionResult,
   ConversationStatus,
   EventHookDescriptor,
+  ImageInput,
   Logger,
   ModelInfo,
   ProviderContract,
@@ -32,11 +33,52 @@ import {
 } from "./pure.js";
 import type { ToolAssembly } from "./tools-filter.js";
 
+// --- Vision handoff (lazy, optional) ---
+
+/**
+ * Minimal contract the vision-handoff service satisfies. Defined here (not
+ * imported from the vision-handoff package) so the orchestrator has NO
+ * compile-time dependency on it — the service is resolved lazily at runtime
+ * (like the message-queue / system-prompt services), and the feature degrades
+ * off cleanly when the extension isn't loaded (images pass through unchanged,
+ * which is correct for vision-capable models and a no-op for text-only turns).
+ *
+ * `transcribeForProvider` transforms a message list for the provider: if the
+ * active model is vision-capable, messages pass through unchanged; otherwise
+ * image chunks are replaced with text descriptions (transcribed via a
+ * vision-capable model). Never throws — degrades to placeholders.
+ */
+export interface VisionHandoffService {
+  readonly transcribeForProvider: (
+    messages: readonly ChatMessage[],
+    currentModelName: string | undefined,
+    opts?: { readonly signal?: AbortSignal; readonly logger?: Logger },
+  ) => Promise<readonly ChatMessage[]>;
+}
+
+/**
+ * Local handle for the vision-handoff service, keyed by the same ID the
+ * vision-handoff extension registers under (`"vision-handoff/service"`). Defined
+ * locally (not imported) so the orchestrator has no compile-time dependency on
+ * the vision-handoff package — the service is resolved lazily at runtime, and
+ * the feature degrades off cleanly when the extension isn't loaded.
+ */
+export const visionHandoffLocalHandle: ServiceHandle<VisionHandoffService> =
+  defineService<VisionHandoffService>("vision-handoff/service");
+
 // --- Broadcast hub types ---
 
 export interface StartTurnInput {
   readonly conversationId: string;
   readonly text: string;
+  /**
+   * Images attached to this turn (e.g. user-pasted screenshots). Each is
+   * appended as an `image` chunk on the persisted user message. For a
+   * vision-capable model the images pass through to the provider natively; for
+   * a non-vision model the vision handoff transcribes them to text first.
+   * Optional — omit for a text-only turn.
+   */
+  readonly images?: readonly ImageInput[];
   readonly modelName?: string;
   readonly cwd?: string;
   /**
@@ -75,6 +117,12 @@ export type StartTurnResult =
 export interface EnqueueInput {
   readonly conversationId: string;
   readonly text: string;
+  /**
+   * Images attached (the steering / opening message analog of
+   * `StartTurnInput.images`). Threaded to `startTurn` when the conversation is
+   * idle (the message starts a turn). Additive optional.
+   */
+  readonly images?: readonly ImageInput[];
   /** Workspace to stamp on a new conversation. Defaults to `"default"`. */
   readonly workspaceId?: string;
   /**
@@ -289,6 +337,8 @@ export interface SessionOrchestrator {
     workspaceId?: string;
     /** Explicit system-prompt override — see {@link StartTurnInput.systemPrompt}. */
     systemPrompt?: string;
+    /** Images attached to this turn — see {@link StartTurnInput.images}. */
+    images?: readonly ImageInput[];
   }): Promise<void>;
 }
 
@@ -335,6 +385,17 @@ export interface SessionOrchestratorDeps {
    * order doesn't matter.
    */
   readonly resolveSystemPrompt?: () => SystemPromptService | undefined;
+  /**
+   * Lazily resolves the vision-handoff service, or `undefined` when the
+   * vision-handoff extension isn't loaded. Used to transcribe image chunks to
+   * text for non-vision models before they reach the provider (so a text-only
+   * model can still reason about pasted/code images). When `undefined`, images
+   * pass through unchanged (correct for vision-capable models; a text-only model
+   * would then receive image content its API may reject — the feature degrades
+   * off cleanly for text-only turns since there are no images). Lazy so
+   * activation order doesn't matter; called per-turn.
+   */
+  readonly resolveVisionHandoff?: () => VisionHandoffService | undefined;
   /** Apply the per-turn tools filter chain. Injected for testability. */
   readonly applyToolsFilter: (assembly: ToolAssembly) => Promise<ToolAssembly>;
   /** Base logger (auto-scoped to this extension); childed per turn for span capture. */
@@ -437,6 +498,7 @@ export function createSessionOrchestrator(
     reasoningEffortOverride: ReasoningEffort | undefined,
     workspaceId: string,
     systemPromptOverride: string | undefined,
+    images: readonly ImageInput[] | undefined,
   ): void {
     const turnId = generateTurnId();
     const controller = new AbortController();
@@ -558,7 +620,7 @@ export function createSessionOrchestrator(
         const effectiveModelName = resolveModelName(modelName, storedModel);
 
         const history = await deps.conversationStore.load(conversationId);
-        const userMsg = buildUserMessage(text);
+        const userMsg = buildUserMessage(text, images);
 
         // Workspace assignment for new conversations happens BEFORE
         // effective-cwd resolution (see workspaceSetupPromise above) so
@@ -697,9 +759,32 @@ export function createSessionOrchestrator(
                 return [{ role: "user", chunks: [{ type: "text", text: steerText }] }];
               };
 
+        // Vision handoff: transform the message list for the provider. When the
+        // active model is vision-capable, images pass through natively (no-op).
+        // When it is NOT vision-capable, image chunks are transcribed to text
+        // descriptions via a vision-capable model — so a text-only model can
+        // still reason about images. The PERSISTED user message keeps the
+        // original image chunks (appended below); only the provider's view is
+        // transcribed. When the vision-handoff service isn't loaded, images pass
+        // through unchanged (correct for vision models; text-only models would
+        // then receive image content their API may reject — degrades off cleanly
+        // for text-only turns with no images).
+        const visionHandoff = deps.resolveVisionHandoff?.();
+        let providerMessages: readonly ChatMessage[] = [...history, userMsg];
+        if (visionHandoff !== undefined) {
+          providerMessages = await visionHandoff.transcribeForProvider(
+            providerMessages,
+            effectiveModelName,
+            {
+              signal: controller.signal,
+              ...(turnLogger !== undefined ? { logger: turnLogger } : {}),
+            },
+          );
+        }
+
         const opts: RunTurnInput = {
           provider,
-          messages: [...history, userMsg],
+          messages: providerMessages,
           tools: assembled.tools,
           dispatch,
           emit: emitAndAccumulate,
@@ -805,6 +890,7 @@ export function createSessionOrchestrator(
       reasoningEffort,
       workspaceId,
       systemPrompt,
+      images,
     }) {
       if (activeTurns.has(conversationId)) {
         return { started: false, reason: "already-active" };
@@ -818,18 +904,20 @@ export function createSessionOrchestrator(
         reasoningEffort,
         workspaceId ?? "default",
         systemPrompt,
+        images,
       );
       const turn = activeTurns.get(conversationId);
       const turnId = turn !== undefined ? turn.turnId : "";
       return { started: true, turnId };
     },
 
-    enqueue({ conversationId, text, workspaceId, computerId }) {
+    enqueue({ conversationId, text, workspaceId, computerId, images }) {
       const result = orchestrator.startTurn({
         conversationId,
         text,
         ...(workspaceId !== undefined ? { workspaceId } : {}),
         ...(computerId !== undefined ? { computerId } : {}),
+        ...(images !== undefined ? { images } : {}),
       });
       if (result.started) {
         return { startedTurn: true, queue: [] };
@@ -914,6 +1002,7 @@ export function createSessionOrchestrator(
       reasoningEffort,
       workspaceId,
       systemPrompt,
+      images,
     }) {
       const turnInput: StartTurnInput = {
         conversationId,
@@ -924,6 +1013,7 @@ export function createSessionOrchestrator(
         ...(reasoningEffort !== undefined ? { reasoningEffort } : {}),
         ...(workspaceId !== undefined ? { workspaceId } : {}),
         ...(systemPrompt !== undefined ? { systemPrompt } : {}),
+        ...(images !== undefined ? { images } : {}),
       };
       const result = orchestrator.startTurn(turnInput);
       if (!result.started) {
diff --git a/packages/session-orchestrator/src/pure.test.ts b/packages/session-orchestrator/src/pure.test.ts
index c75cb82..7a574f1 100644
--- a/packages/session-orchestrator/src/pure.test.ts
+++ b/packages/session-orchestrator/src/pure.test.ts
@@ -26,6 +26,39 @@ describe("buildUserMessage", () => {
     expect(msg.role).toBe("user");
     expect(msg.chunks[0]).toEqual({ type: "text", text: "" });
   });
+
+  it("appends image chunks after the text chunk when images are given", () => {
+    const msg = buildUserMessage("look at this", [
+      { url: "data:image/png;base64,aaa" },
+      { url: "data:image/jpeg;base64,bbb", mimeType: "image/jpeg" },
+    ]);
+    expect(msg.chunks).toHaveLength(3);
+    expect(msg.chunks[0]).toEqual({ type: "text", text: "look at this" });
+    expect(msg.chunks[1]).toEqual({ type: "image", url: "data:image/png;base64,aaa" });
+    expect(msg.chunks[2]).toEqual({
+      type: "image",
+      url: "data:image/jpeg;base64,bbb",
+      mimeType: "image/jpeg",
+    });
+  });
+
+  it("builds an image-only message when text is empty", () => {
+    const msg = buildUserMessage("", [{ url: "data:image/png;base64,zzz" }]);
+    expect(msg.chunks).toHaveLength(1);
+    expect(msg.chunks[0]).toEqual({ type: "image", url: "data:image/png;base64,zzz" });
+  });
+
+  it("includes mimeType when provided", () => {
+    const msg = buildUserMessage("hi", [
+      { url: "data:image/webp;base64,x", mimeType: "image/webp" },
+    ]);
+    expect((msg.chunks[1] as { mimeType?: string }).mimeType).toBe("image/webp");
+  });
+
+  it("omits mimeType when not provided", () => {
+    const msg = buildUserMessage("hi", [{ url: "https://example.com/x.png" }]);
+    expect((msg.chunks[1] as { mimeType?: string }).mimeType).toBeUndefined();
+  });
 });
 
 describe("selectFirstProvider", () => {
diff --git a/packages/session-orchestrator/src/pure.ts b/packages/session-orchestrator/src/pure.ts
index 2208e8f..0d2068f 100644
--- a/packages/session-orchestrator/src/pure.ts
+++ b/packages/session-orchestrator/src/pure.ts
@@ -1,12 +1,40 @@
 import type {
   ChatMessage,
+  Chunk,
+  ImageInput,
   ProviderContract,
   ReasoningEffort,
   ToolDispatchPolicy,
 } from "@dispatch/kernel";
 
-export function buildUserMessage(text: string): ChatMessage {
-  return { role: "user", chunks: [{ type: "text", text }] };
+/**
+ * Build the persisted user message for a turn. When `images` are provided, each
+ * is appended as an `image` chunk AFTER the text chunk, so the persisted message
+ * carries both the prompt text and the attached images (the frontend renders
+ * the images; vision-capable providers receive them natively; non-vision
+ * providers have them transcribed by the vision handoff before streaming).
+ *
+ * Pure: inputs → a ChatMessage, no I/O.
+ */
+export function buildUserMessage(text: string, images?: readonly ImageInput[]): ChatMessage {
+  const chunks: Chunk[] = [];
+  if (text.length > 0) {
+    chunks.push({ type: "text", text });
+  }
+  if (images !== undefined) {
+    for (const img of images) {
+      chunks.push({
+        type: "image",
+        url: img.url,
+        ...(img.mimeType !== undefined ? { mimeType: img.mimeType } : {}),
+      });
+    }
+  }
+  // An image-only message (empty text) is valid.
+  if (chunks.length === 0) {
+    chunks.push({ type: "text", text: "" });
+  }
+  return { role: "user", chunks };
 }
 
 // ── Provider-error retry backoff schedule ───────────────────────────────────