feat(vision): image compaction for vision-capable models + global vision settings

12 files changed, 318 insertions, 16 deletions

diff --git a/bun.lock b/bun.lock
index 8a913d0..d9762f7 100644
--- a/bun.lock
+++ b/bun.lock
@@ -366,6 +366,7 @@
       "name": "@dispatch/vision-handoff",
       "version": "0.0.0",
       "dependencies": {
+        "@dispatch/conversation-store": "workspace:*",
         "@dispatch/credential-store": "workspace:*",
         "@dispatch/kernel": "workspace:*",
         "@dispatch/openai-stream": "workspace:*",
diff --git a/packages/conversation-store/src/keys.ts b/packages/conversation-store/src/keys.ts
index b2c635d..6ec2bc5 100644
--- a/packages/conversation-store/src/keys.ts
+++ b/packages/conversation-store/src/keys.ts
@@ -66,6 +66,14 @@ export function compactThresholdKey(conversationId: string): string {
   return `conv:${conversationId}:compact-percent`;
 }
 
+/** Per-conversation image transcription cache (JSON map of imageUrl → transcription). */
+export function imageTranscriptionsKey(conversationId: string): string {
+  return `conv:${conversationId}:image-transcriptions`;
+}
+
+/** Global vision settings (image compaction limit + compaction model). */
+export const VISION_SETTINGS_KEY = "vision-settings";
+
 export function metaKey(conversationId: string): string {
   return `conv:${conversationId}:meta`;
 }
diff --git a/packages/conversation-store/src/store.ts b/packages/conversation-store/src/store.ts
index f90e809..69334e6 100644
--- a/packages/conversation-store/src/store.ts
+++ b/packages/conversation-store/src/store.ts
@@ -20,6 +20,7 @@ import {
   compactThresholdKey,
   computerKey,
   cwdKey,
+  imageTranscriptionsKey,
   metaKey,
   metricsKey,
   metricsPrefix,
@@ -28,6 +29,7 @@ import {
   parseSeq,
   reasoningEffortKey,
   seqKey,
+  VISION_SETTINGS_KEY,
   workspaceKey,
 } from "./keys.js";
 import { reconcileWithReport } from "./reconcile.js";
@@ -141,6 +143,35 @@ export interface ConversationStore {
   /** Set the compact percent (0-100, 0 = manual only). */
   readonly setCompactPercent: (conversationId: string, percent: number) => Promise<void>;
   /**
+   * Get the per-conversation image transcription cache: a map of image URL →
+   * transcription text. Used by the vision handoff to avoid re-transcribing
+   * old images that were compacted to text on a previous turn. Returns an
+   * empty map when none are cached.
+   */
+  readonly getImageTranscriptions: (conversationId: string) => Promise<ReadonlyMap<string, string>>;
+  /**
+   * Upsert a single image transcription into the per-conversation cache.
+   * Merges with any existing transcriptions (does NOT replace the whole map).
+   */
+  readonly setImageTranscription: (
+    conversationId: string,
+    imageUrl: string,
+    transcription: string,
+  ) => Promise<void>;
+  /**
+   * Get the global vision settings (image compaction limit + compaction model).
+   * The limit defaults to 10 when never set; the compaction model defaults to
+   * null (auto-select). Shared across ALL conversations and vision models.
+   */
+  readonly getVisionSettings: () => Promise<{
+    readonly imageLimit: number;
+    readonly compactionModel: string | null;
+  }>;
+  /** Set the global vision image compaction limit (0 = disabled). */
+  readonly setVisionImageLimit: (limit: number) => Promise<void>;
+  /** Set the global vision compaction model (null = auto-select). */
+  readonly setVisionCompactionModel: (model: string | null) => Promise<void>;
+  /**
    * Set the `compactedFrom` field on a conversation's metadata, pointing to
    * the archive conversation that holds the pre-compaction history.
    */
@@ -1004,6 +1035,52 @@ export function createConversationStore(
       }
     },
 
+    async getImageTranscriptions(conversationId) {
+      const raw = await storage.get(imageTranscriptionsKey(conversationId));
+      if (raw === null) return new Map();
+      try {
+        const obj = JSON.parse(raw) as Record<string, string>;
+        return new Map(Object.entries(obj));
+      } catch {
+        return new Map();
+      }
+    },
+
+    async setImageTranscription(conversationId, imageUrl, transcription) {
+      const existing = await this.getImageTranscriptions(conversationId);
+      const merged = new Map(existing);
+      merged.set(imageUrl, transcription);
+      const obj: Record<string, string> = {};
+      for (const [k, v] of merged) obj[k] = v;
+      await storage.set(imageTranscriptionsKey(conversationId), JSON.stringify(obj));
+    },
+
+    async getVisionSettings() {
+      const raw = await storage.get(VISION_SETTINGS_KEY);
+      if (raw === null) return { imageLimit: 10, compactionModel: null };
+      try {
+        const obj = JSON.parse(raw) as { imageLimit?: number; compactionModel?: string | null };
+        return {
+          imageLimit: typeof obj.imageLimit === "number" ? obj.imageLimit : 10,
+          compactionModel: obj.compactionModel ?? null,
+        };
+      } catch {
+        return { imageLimit: 10, compactionModel: null };
+      }
+    },
+
+    async setVisionImageLimit(limit) {
+      const current = await this.getVisionSettings();
+      const obj = { imageLimit: limit, compactionModel: current.compactionModel };
+      await storage.set(VISION_SETTINGS_KEY, JSON.stringify(obj));
+    },
+
+    async setVisionCompactionModel(model) {
+      const current = await this.getVisionSettings();
+      const obj = { imageLimit: current.imageLimit, compactionModel: model };
+      await storage.set(VISION_SETTINGS_KEY, JSON.stringify(obj));
+    },
+
     async setCompactedFrom(conversationId, newConversationId) {
       const raw = await storage.get(metaKey(conversationId));
       const existing = raw !== null ? parseMetaRow(raw) : null;
diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts
index 4f4bb3e..c0493f3 100644
--- a/packages/session-orchestrator/src/orchestrator.ts
+++ b/packages/session-orchestrator/src/orchestrator.ts
@@ -54,6 +54,7 @@ export interface VisionHandoffService {
     currentModelName: string | undefined,
     opts?: {
       readonly conversationId?: string;
+      readonly imageLimit?: number;
       readonly signal?: AbortSignal;
       readonly logger?: Logger;
     },
@@ -776,11 +777,13 @@ export function createSessionOrchestrator(
         const visionHandoff = deps.resolveVisionHandoff?.();
         let providerMessages: readonly ChatMessage[] = [...history, userMsg];
         if (visionHandoff !== undefined) {
+          const visionSettings = await deps.conversationStore.getVisionSettings();
           providerMessages = await visionHandoff.prepareForProvider(
             providerMessages,
             effectiveModelName,
             {
               conversationId,
+              imageLimit: visionSettings.imageLimit,
               signal: controller.signal,
               ...(turnLogger !== undefined ? { logger: turnLogger } : {}),
             },
diff --git a/packages/transport-contract/src/index.ts b/packages/transport-contract/src/index.ts
index 0444f29..94897f7 100644
--- a/packages/transport-contract/src/index.ts
+++ b/packages/transport-contract/src/index.ts
@@ -411,6 +411,23 @@ export interface SystemPromptVariablesResponse {
   readonly variables: readonly SystemPromptVariable[];
 }
 
+// ─── Vision settings (global) ──────────────────────────────────────────────────
+
+/**
+ * Response of `GET /settings/vision` — the global vision configuration shared
+ * across all conversations and vision models.
+ */
+export interface VisionSettingsResponse {
+  readonly imageLimit: number;
+  readonly compactionModel: string | null;
+}
+
+/** Body of `PUT /settings/vision` — a partial update. */
+export interface SetVisionSettingsRequest {
+  readonly imageLimit?: number;
+  readonly compactionModel?: string | null;
+}
+
 // ─── Message queue (steering) ─────────────────────────────────────────────────
 
 /**
diff --git a/packages/transport-http/src/app.ts b/packages/transport-http/src/app.ts
index a9a23da..ea216e1 100644
--- a/packages/transport-http/src/app.ts
+++ b/packages/transport-http/src/app.ts
@@ -38,6 +38,7 @@ import type {
   ThroughputResponse,
   TitleResponse,
   UpdateHeartbeatRequest,
+  VisionSettingsResponse,
   WarmResponse,
   WorkspaceListResponse,
   WorkspaceResponse,
@@ -1594,6 +1595,43 @@ export function createApp(opts: CreateServerOptions): Hono {
     return c.json(response, 200);
   });
 
+  app.get("/settings/vision", async (c) => {
+    const settings = await opts.conversationStore.getVisionSettings();
+    const body: VisionSettingsResponse = settings;
+    return c.json(body, 200);
+  });
+
+  app.put("/settings/vision", async (c) => {
+    let body: unknown;
+    try {
+      body = await c.req.json();
+    } catch {
+      return c.json({ error: "Invalid JSON body" }, 400);
+    }
+    const obj = body as { imageLimit?: unknown; compactionModel?: unknown };
+    if (obj.imageLimit !== undefined) {
+      if (
+        typeof obj.imageLimit !== "number" ||
+        !Number.isInteger(obj.imageLimit) ||
+        obj.imageLimit < 0
+      ) {
+        return c.json({ error: "imageLimit must be a non-negative integer" }, 400);
+      }
+      await opts.conversationStore.setVisionImageLimit(obj.imageLimit);
+      log.info("vision: image limit set", { imageLimit: obj.imageLimit });
+    }
+    if (obj.compactionModel !== undefined) {
+      if (obj.compactionModel !== null && typeof obj.compactionModel !== "string") {
+        return c.json({ error: "compactionModel must be a string or null" }, 400);
+      }
+      await opts.conversationStore.setVisionCompactionModel(obj.compactionModel);
+      log.info("vision: compaction model set", { compactionModel: obj.compactionModel });
+    }
+    const settings = await opts.conversationStore.getVisionSettings();
+    const response: VisionSettingsResponse = settings;
+    return c.json(response, 200);
+  });
+
   // ─── Static frontend serving (catch-all, API routes take precedence) ──────
   if (opts.webDir !== undefined) {
     const webDir = opts.webDir;
diff --git a/packages/vision-handoff/package.json b/packages/vision-handoff/package.json
index a88ab49..b11f7ee 100644
--- a/packages/vision-handoff/package.json
+++ b/packages/vision-handoff/package.json
@@ -6,6 +6,7 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "dependencies": {
+    "@dispatch/conversation-store": "workspace:*",
     "@dispatch/credential-store": "workspace:*",
     "@dispatch/kernel": "workspace:*",
     "@dispatch/openai-stream": "workspace:*"
diff --git a/packages/vision-handoff/src/extension.ts b/packages/vision-handoff/src/extension.ts
index 2f75a6b..af646aa 100644
--- a/packages/vision-handoff/src/extension.ts
+++ b/packages/vision-handoff/src/extension.ts
@@ -16,6 +16,7 @@
 
 import { readFile } from "node:fs/promises";
 import { extname, isAbsolute, resolve as pathResolve } from "node:path";
+import { conversationStoreHandle } from "@dispatch/conversation-store";
 import type { CredentialStore } from "@dispatch/credential-store";
 import { credentialStoreHandle } from "@dispatch/credential-store";
 import type { Extension, HostAPI, Manifest } from "@dispatch/kernel";
@@ -81,10 +82,6 @@ export async function activate(host: HostAPI): Promise<void> {
     credentialStore,
     resolveModel,
     readFileAsDataUrl,
-    // Lazily resolve the session-orchestrator (for starting vision consultation
-    // turns). By the time consult_vision is called at runtime, all extensions
-    // have activated. The activated-manifests guard avoids a getService throw
-    // when the orchestrator isn't loaded.
     resolveOrchestrator: () => {
       const loaded = host.getExtensions().some((m) => m.id === "session-orchestrator");
       if (!loaded) return undefined;
@@ -94,6 +91,14 @@ export async function activate(host: HostAPI): Promise<void> {
         return undefined;
       }
     },
+    getImageTranscriptions: async (conversationId: string) => {
+      const store = host.getService(conversationStoreHandle);
+      return store.getImageTranscriptions(conversationId);
+    },
+    setImageTranscription: async (conversationId: string, url: string, text: string) => {
+      const store = host.getService(conversationStoreHandle);
+      await store.setImageTranscription(conversationId, url, text);
+    },
     logger: host.logger.child({ extensionId: "vision-handoff" }),
   });
 
diff --git a/packages/vision-handoff/src/pure.test.ts b/packages/vision-handoff/src/pure.test.ts
index ea28288..e3fcf58 100644
--- a/packages/vision-handoff/src/pure.test.ts
+++ b/packages/vision-handoff/src/pure.test.ts
@@ -11,11 +11,15 @@ import {
 
 describe("isVisionCapable", () => {
   it("returns true when ModelInfo.vision is true", () => {
-    expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: true })).toBe(true);
+    expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: true })).toBe(
+      true,
+    );
   });
 
   it("returns false when ModelInfo.vision is false (overrides name heuristic)", () => {
-    expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: false })).toBe(false);
+    expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: false })).toBe(
+      false,
+    );
   });
 
   it("falls back to name heuristic when vision is absent (umans kimi + qwen)", () => {
@@ -53,10 +57,7 @@ describe("findVisionModelName", () => {
   });
 
   it("finds a vision model via ModelInfo.vision when name heuristic misses", async () => {
-    const name = await findVisionModelName(
-      ["umans/umans-glm-5.2", "umans/llama-vision"],
-      getInfo,
-    );
+    const name = await findVisionModelName(["umans/umans-glm-5.2", "umans/llama-vision"], getInfo);
     expect(name).toBe("umans/llama-vision");
   });
 
diff --git a/packages/vision-handoff/src/service.test.ts b/packages/vision-handoff/src/service.test.ts
index dc54902..4667dbc 100644
--- a/packages/vision-handoff/src/service.test.ts
+++ b/packages/vision-handoff/src/service.test.ts
@@ -47,7 +47,8 @@ function makeDeps(overrides: Partial<VisionHandoffDeps> = {}): VisionHandoffDeps
       listCatalog: vi.fn(async () => catalog),
       getModelInfo: vi.fn(async (name: string) => infoMap[name]),
       resolve: vi.fn((name: string) => {
-        if (name === "umans/umans-kimi-k2.7") return { providerId: "umans", model: "umans-kimi-k2.7" };
+        if (name === "umans/umans-kimi-k2.7")
+          return { providerId: "umans", model: "umans-kimi-k2.7" };
         if (name === "umans/umans-glm-5.2") return { providerId: "umans", model: "umans-glm-5.2" };
         return undefined;
       }),
diff --git a/packages/vision-handoff/src/service.ts b/packages/vision-handoff/src/service.ts
index 78f241f..7403c21 100644
--- a/packages/vision-handoff/src/service.ts
+++ b/packages/vision-handoff/src/service.ts
@@ -97,6 +97,24 @@ export interface VisionHandoffDeps {
    * with an error. Lazy so activation order doesn't matter.
    */
   readonly resolveOrchestrator?: () => OrchestratorForVision | undefined;
+  /**
+   * Get the per-conversation cached image transcriptions (imageUrl → text).
+   * Used to avoid re-transcribing old images that were compacted to text on a
+   * previous turn. Optional — when absent, compaction still works but
+   * re-transcribes every turn (no caching).
+   */
+  readonly getImageTranscriptions?: (
+    conversationId: string,
+  ) => Promise<ReadonlyMap<string, string>>;
+  /**
+   * Upsert a single image transcription into the per-conversation cache.
+   * Optional — paired with getImageTranscriptions.
+   */
+  readonly setImageTranscription?: (
+    conversationId: string,
+    imageUrl: string,
+    transcription: string,
+  ) => Promise<void>;
   /** Generate a new conversation ID for a consultation. Defaults to crypto.randomUUID. */
   readonly generateId?: () => string;
   readonly logger?: Logger;
@@ -128,6 +146,7 @@ export interface VisionHandoffService {
     currentModelName: string | undefined,
     opts?: {
       readonly conversationId?: string;
+      readonly imageLimit?: number;
       readonly signal?: AbortSignal;
       readonly logger?: Logger;
     },
@@ -198,6 +217,129 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando
     return { provider: resolved.provider, model: resolved.model, modelName: name };
   }
 
+  /**
+   * Compact images for a vision-capable model: when the conversation has more
+   * image chunks than the limit, the oldest images are transcribed to text
+   * (one-time, cached in the conversation store) and stripped from the
+   * provider messages. Recent images (within the limit) stay native.
+   *
+   * The persisted history is NOT modified — only the provider's view.
+   * Transcriptions are cached so they're reused on subsequent turns (no
+   * re-transcription). When no caching deps are available, it still works but
+   * re-transcribes every turn.
+   */
+  async function compactImagesForVisionModel(
+    messages: readonly ChatMessage[],
+    opts:
+      | {
+          readonly conversationId?: string;
+          readonly imageLimit?: number;
+          readonly signal?: AbortSignal;
+          readonly logger?: Logger;
+        }
+      | undefined,
+    currentModelName: string | undefined,
+  ): Promise<readonly ChatMessage[]> {
+    void currentModelName; // reserved for future model-specific compaction logic
+    const limit = opts?.imageLimit;
+    // No limit or limit <= 0 → pass all images through (compaction disabled).
+    if (limit === undefined || limit <= 0) return messages;
+
+    // Collect all image chunks in order (oldest first, across all messages).
+    const imageEntries: { msgIdx: number; chunkIdx: number; url: string }[] = [];
+    for (const [mi, msg] of messages.entries()) {
+      for (const [ci, chunk] of msg.chunks.entries()) {
+        if (chunk.type === "image") {
+          imageEntries.push({ msgIdx: mi, chunkIdx: ci, url: chunk.url });
+        }
+      }
+    }
+
+    // If within the limit, pass everything through natively.
+    if (imageEntries.length <= limit) return messages;
+
+    // The oldest (imageEntries.length - limit) images need transcription.
+    const toTranscribeCount = imageEntries.length - limit;
+    const toTranscribe = imageEntries.slice(0, toTranscribeCount);
+
+    // Load cached transcriptions.
+    const convId = opts?.conversationId;
+    const cache =
+      convId !== undefined && deps.getImageTranscriptions !== undefined
+        ? await deps.getImageTranscriptions(convId)
+        : new Map<string, string>();
+
+    // Transcribe any that aren't cached yet (via the vision model).
+    const transcriptions = new Map<string, string>(cache);
+    const vision = await resolveVisionModel();
+    for (const entry of toTranscribe) {
+      if (transcriptions.has(entry.url)) continue;
+      if (vision === undefined) {
+        // No vision model available for transcription — use a placeholder.
+        transcriptions.set(
+          entry.url,
+          "[Image was compacted — no vision model available to transcribe it.]",
+        );
+        continue;
+      }
+      try {
+        const prompt =
+          "Describe this image in detail. Include visible text (transcribe verbatim), " +
+          "key objects, layout, and notable details. This description will replace " +
+          "the image in a conversation history, so be thorough.";
+        const userMessage: ChatMessage = {
+          role: "user",
+          chunks: [
+            { type: "text", text: prompt },
+            { type: "image", url: entry.url },
+          ],
+        };
+        const stream = vision.provider.stream([userMessage], [], {
+          model: vision.model,
+          systemPrompt: "You are a vision assistant. Describe images faithfully and thoroughly.",
+        });
+        const description = (await collectTextFromStream(stream)).trim();
+        const text =
+          description.length > 0 ? description : "[Image transcription produced no output.]";
+        transcriptions.set(entry.url, text);
+        // Cache it in the conversation store (if available).
+        if (convId !== undefined && deps.setImageTranscription !== undefined) {
+          await deps.setImageTranscription(convId, entry.url, text);
+        }
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        log?.warn("vision-handoff: image compaction transcription failed", { error: msg });
+        transcriptions.set(entry.url, `[Image transcription failed: ${msg}]`);
+      }
+    }
+
+    // Build the provider messages: replace transcribed images with text,
+    // keep recent images (within the limit) native.
+    const transcribedUrls = new Set(toTranscribe.map((e) => e.url));
+    const result: ChatMessage[] = [];
+    for (const msg of messages) {
+      if (!msg.chunks.some((c) => c.type === "image")) {
+        result.push(msg);
+        continue;
+      }
+      const newChunks: Chunk[] = [];
+      for (const chunk of msg.chunks) {
+        if (chunk.type === "image" && transcribedUrls.has(chunk.url)) {
+          const transcription = transcriptions.get(chunk.url);
+          if (transcription !== undefined) {
+            newChunks.push({ type: "text", text: `[Compacted image]: ${transcription}` });
+          } else {
+            newChunks.push(chunk); // fallback: keep the image
+          }
+        } else {
+          newChunks.push(chunk);
+        }
+      }
+      result.push({ role: msg.role, chunks: newChunks });
+    }
+    return result;
+  }
+
   const service: VisionHandoffService = {
     async isVisionCapable(modelName: string | undefined): Promise<boolean> {
       if (modelName === undefined) return false;
@@ -212,6 +354,7 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando
       currentModelName: string | undefined,
       opts?: {
         readonly conversationId?: string;
+        readonly imageLimit?: number;
         readonly signal?: AbortSignal;
         readonly logger?: Logger;
       },
@@ -219,13 +362,19 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando
       // Fast path: no images anywhere → nothing to do.
       if (!hasImageChunks(messages)) return messages;
 
-      // If the active model IS vision-capable, pass images through natively.
-      if (currentModelName !== undefined) {
-        const capable = await isVisionCapable(currentModelName, await getInfo(currentModelName));
-        if (capable) return messages;
+      const isCapable =
+        currentModelName !== undefined &&
+        (await isVisionCapable(currentModelName, await getInfo(currentModelName)));
+
+      // ── Vision-capable model: image compaction ──────────────────────────
+      // When the conversation has more images than the limit, the oldest images
+      // are transcribed to text (one-time, cached) and stripped from the
+      // provider messages. Recent images (within the limit) stay native.
+      if (isCapable) {
+        return compactImagesForVisionModel(messages, opts, currentModelName);
       }
 
-      // Non-vision model: check if a vision model is available at all.
+      // ── Non-vision model: placeholders + consult_vision ──────────────────
       const vision = await resolveVisionModel();
       const convId = opts?.conversationId;
 
diff --git a/packages/vision-handoff/tsconfig.json b/packages/vision-handoff/tsconfig.json
index ec597fc..b5439aa 100644
--- a/packages/vision-handoff/tsconfig.json
+++ b/packages/vision-handoff/tsconfig.json
@@ -5,6 +5,7 @@
   "references": [
     { "path": "../kernel" },
     { "path": "../wire" },
+    { "path": "../conversation-store" },
     { "path": "../credential-store" },
     { "path": "../openai-stream" }
   ]