From d5633cf6e007eaf8255a44529a638d2466a74ba3 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 03:40:38 +0900 Subject: feat(vision-handoff): implement vision for capable models and universal vision handoff --- bun.lock | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'bun.lock') diff --git a/bun.lock b/bun.lock index 2261ba8..8a913d0 100644 --- a/bun.lock +++ b/bun.lock @@ -103,6 +103,7 @@ "@dispatch/tool-youtube-transcript": "workspace:*", "@dispatch/transport-http": "workspace:*", "@dispatch/transport-ws": "workspace:*", + "@dispatch/vision-handoff": "workspace:*", }, }, "packages/journal-sink": { @@ -361,6 +362,15 @@ "name": "@dispatch/ui-contract", "version": "0.2.0", }, + "packages/vision-handoff": { + "name": "@dispatch/vision-handoff", + "version": "0.0.0", + "dependencies": { + "@dispatch/credential-store": "workspace:*", + "@dispatch/kernel": "workspace:*", + "@dispatch/openai-stream": "workspace:*", + }, + }, "packages/wire": { "name": "@dispatch/wire", "version": "0.12.0", @@ -461,6 +471,8 @@ "@dispatch/ui-contract": ["@dispatch/ui-contract@workspace:packages/ui-contract"], + "@dispatch/vision-handoff": ["@dispatch/vision-handoff@workspace:packages/vision-handoff"], + "@dispatch/wire": ["@dispatch/wire@workspace:packages/wire"], "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.27.7", "", { "os": "aix", "cpu": "ppc64" }, "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg=="], -- cgit v1.2.3 From 2c91dc63802a386b1612ea0ed8c1e96b6f4421db Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 18:46:56 +0900 Subject: feat(vision): image compaction for vision-capable models + global vision settings --- bun.lock | 1 + packages/conversation-store/src/keys.ts | 8 ++ packages/conversation-store/src/store.ts | 77 +++++++++++ packages/session-orchestrator/src/orchestrator.ts | 3 + packages/transport-contract/src/index.ts | 17 +++ packages/transport-http/src/app.ts | 38 ++++++ packages/vision-handoff/package.json | 1 + packages/vision-handoff/src/extension.ts | 13 +- packages/vision-handoff/src/pure.test.ts | 13 +- packages/vision-handoff/src/service.test.ts | 3 +- packages/vision-handoff/src/service.ts | 159 +++++++++++++++++++++- packages/vision-handoff/tsconfig.json | 1 + 12 files changed, 318 insertions(+), 16 deletions(-) (limited to 'bun.lock') diff --git a/bun.lock b/bun.lock index 8a913d0..d9762f7 100644 --- a/bun.lock +++ b/bun.lock @@ -366,6 +366,7 @@ "name": "@dispatch/vision-handoff", "version": "0.0.0", "dependencies": { + "@dispatch/conversation-store": "workspace:*", "@dispatch/credential-store": "workspace:*", "@dispatch/kernel": "workspace:*", "@dispatch/openai-stream": "workspace:*", diff --git a/packages/conversation-store/src/keys.ts b/packages/conversation-store/src/keys.ts index b2c635d..6ec2bc5 100644 --- a/packages/conversation-store/src/keys.ts +++ b/packages/conversation-store/src/keys.ts @@ -66,6 +66,14 @@ export function compactThresholdKey(conversationId: string): string { return `conv:${conversationId}:compact-percent`; } +/** Per-conversation image transcription cache (JSON map of imageUrl → transcription). */ +export function imageTranscriptionsKey(conversationId: string): string { + return `conv:${conversationId}:image-transcriptions`; +} + +/** Global vision settings (image compaction limit + compaction model). */ +export const VISION_SETTINGS_KEY = "vision-settings"; + export function metaKey(conversationId: string): string { return `conv:${conversationId}:meta`; } diff --git a/packages/conversation-store/src/store.ts b/packages/conversation-store/src/store.ts index f90e809..69334e6 100644 --- a/packages/conversation-store/src/store.ts +++ b/packages/conversation-store/src/store.ts @@ -20,6 +20,7 @@ import { compactThresholdKey, computerKey, cwdKey, + imageTranscriptionsKey, metaKey, metricsKey, metricsPrefix, @@ -28,6 +29,7 @@ import { parseSeq, reasoningEffortKey, seqKey, + VISION_SETTINGS_KEY, workspaceKey, } from "./keys.js"; import { reconcileWithReport } from "./reconcile.js"; @@ -140,6 +142,35 @@ export interface ConversationStore { readonly getCompactPercent: (conversationId: string) => Promise; /** Set the compact percent (0-100, 0 = manual only). */ readonly setCompactPercent: (conversationId: string, percent: number) => Promise; + /** + * Get the per-conversation image transcription cache: a map of image URL → + * transcription text. Used by the vision handoff to avoid re-transcribing + * old images that were compacted to text on a previous turn. Returns an + * empty map when none are cached. + */ + readonly getImageTranscriptions: (conversationId: string) => Promise>; + /** + * Upsert a single image transcription into the per-conversation cache. + * Merges with any existing transcriptions (does NOT replace the whole map). + */ + readonly setImageTranscription: ( + conversationId: string, + imageUrl: string, + transcription: string, + ) => Promise; + /** + * Get the global vision settings (image compaction limit + compaction model). + * The limit defaults to 10 when never set; the compaction model defaults to + * null (auto-select). Shared across ALL conversations and vision models. + */ + readonly getVisionSettings: () => Promise<{ + readonly imageLimit: number; + readonly compactionModel: string | null; + }>; + /** Set the global vision image compaction limit (0 = disabled). */ + readonly setVisionImageLimit: (limit: number) => Promise; + /** Set the global vision compaction model (null = auto-select). */ + readonly setVisionCompactionModel: (model: string | null) => Promise; /** * Set the `compactedFrom` field on a conversation's metadata, pointing to * the archive conversation that holds the pre-compaction history. @@ -1004,6 +1035,52 @@ export function createConversationStore( } }, + async getImageTranscriptions(conversationId) { + const raw = await storage.get(imageTranscriptionsKey(conversationId)); + if (raw === null) return new Map(); + try { + const obj = JSON.parse(raw) as Record; + return new Map(Object.entries(obj)); + } catch { + return new Map(); + } + }, + + async setImageTranscription(conversationId, imageUrl, transcription) { + const existing = await this.getImageTranscriptions(conversationId); + const merged = new Map(existing); + merged.set(imageUrl, transcription); + const obj: Record = {}; + for (const [k, v] of merged) obj[k] = v; + await storage.set(imageTranscriptionsKey(conversationId), JSON.stringify(obj)); + }, + + async getVisionSettings() { + const raw = await storage.get(VISION_SETTINGS_KEY); + if (raw === null) return { imageLimit: 10, compactionModel: null }; + try { + const obj = JSON.parse(raw) as { imageLimit?: number; compactionModel?: string | null }; + return { + imageLimit: typeof obj.imageLimit === "number" ? obj.imageLimit : 10, + compactionModel: obj.compactionModel ?? null, + }; + } catch { + return { imageLimit: 10, compactionModel: null }; + } + }, + + async setVisionImageLimit(limit) { + const current = await this.getVisionSettings(); + const obj = { imageLimit: limit, compactionModel: current.compactionModel }; + await storage.set(VISION_SETTINGS_KEY, JSON.stringify(obj)); + }, + + async setVisionCompactionModel(model) { + const current = await this.getVisionSettings(); + const obj = { imageLimit: current.imageLimit, compactionModel: model }; + await storage.set(VISION_SETTINGS_KEY, JSON.stringify(obj)); + }, + async setCompactedFrom(conversationId, newConversationId) { const raw = await storage.get(metaKey(conversationId)); const existing = raw !== null ? parseMetaRow(raw) : null; diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts index 4f4bb3e..c0493f3 100644 --- a/packages/session-orchestrator/src/orchestrator.ts +++ b/packages/session-orchestrator/src/orchestrator.ts @@ -54,6 +54,7 @@ export interface VisionHandoffService { currentModelName: string | undefined, opts?: { readonly conversationId?: string; + readonly imageLimit?: number; readonly signal?: AbortSignal; readonly logger?: Logger; }, @@ -776,11 +777,13 @@ export function createSessionOrchestrator( const visionHandoff = deps.resolveVisionHandoff?.(); let providerMessages: readonly ChatMessage[] = [...history, userMsg]; if (visionHandoff !== undefined) { + const visionSettings = await deps.conversationStore.getVisionSettings(); providerMessages = await visionHandoff.prepareForProvider( providerMessages, effectiveModelName, { conversationId, + imageLimit: visionSettings.imageLimit, signal: controller.signal, ...(turnLogger !== undefined ? { logger: turnLogger } : {}), }, diff --git a/packages/transport-contract/src/index.ts b/packages/transport-contract/src/index.ts index 0444f29..94897f7 100644 --- a/packages/transport-contract/src/index.ts +++ b/packages/transport-contract/src/index.ts @@ -411,6 +411,23 @@ export interface SystemPromptVariablesResponse { readonly variables: readonly SystemPromptVariable[]; } +// ─── Vision settings (global) ────────────────────────────────────────────────── + +/** + * Response of `GET /settings/vision` — the global vision configuration shared + * across all conversations and vision models. + */ +export interface VisionSettingsResponse { + readonly imageLimit: number; + readonly compactionModel: string | null; +} + +/** Body of `PUT /settings/vision` — a partial update. */ +export interface SetVisionSettingsRequest { + readonly imageLimit?: number; + readonly compactionModel?: string | null; +} + // ─── Message queue (steering) ───────────────────────────────────────────────── /** diff --git a/packages/transport-http/src/app.ts b/packages/transport-http/src/app.ts index a9a23da..ea216e1 100644 --- a/packages/transport-http/src/app.ts +++ b/packages/transport-http/src/app.ts @@ -38,6 +38,7 @@ import type { ThroughputResponse, TitleResponse, UpdateHeartbeatRequest, + VisionSettingsResponse, WarmResponse, WorkspaceListResponse, WorkspaceResponse, @@ -1594,6 +1595,43 @@ export function createApp(opts: CreateServerOptions): Hono { return c.json(response, 200); }); + app.get("/settings/vision", async (c) => { + const settings = await opts.conversationStore.getVisionSettings(); + const body: VisionSettingsResponse = settings; + return c.json(body, 200); + }); + + app.put("/settings/vision", async (c) => { + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid JSON body" }, 400); + } + const obj = body as { imageLimit?: unknown; compactionModel?: unknown }; + if (obj.imageLimit !== undefined) { + if ( + typeof obj.imageLimit !== "number" || + !Number.isInteger(obj.imageLimit) || + obj.imageLimit < 0 + ) { + return c.json({ error: "imageLimit must be a non-negative integer" }, 400); + } + await opts.conversationStore.setVisionImageLimit(obj.imageLimit); + log.info("vision: image limit set", { imageLimit: obj.imageLimit }); + } + if (obj.compactionModel !== undefined) { + if (obj.compactionModel !== null && typeof obj.compactionModel !== "string") { + return c.json({ error: "compactionModel must be a string or null" }, 400); + } + await opts.conversationStore.setVisionCompactionModel(obj.compactionModel); + log.info("vision: compaction model set", { compactionModel: obj.compactionModel }); + } + const settings = await opts.conversationStore.getVisionSettings(); + const response: VisionSettingsResponse = settings; + return c.json(response, 200); + }); + // ─── Static frontend serving (catch-all, API routes take precedence) ────── if (opts.webDir !== undefined) { const webDir = opts.webDir; diff --git a/packages/vision-handoff/package.json b/packages/vision-handoff/package.json index a88ab49..b11f7ee 100644 --- a/packages/vision-handoff/package.json +++ b/packages/vision-handoff/package.json @@ -6,6 +6,7 @@ "main": "dist/index.js", "types": "dist/index.d.ts", "dependencies": { + "@dispatch/conversation-store": "workspace:*", "@dispatch/credential-store": "workspace:*", "@dispatch/kernel": "workspace:*", "@dispatch/openai-stream": "workspace:*" diff --git a/packages/vision-handoff/src/extension.ts b/packages/vision-handoff/src/extension.ts index 2f75a6b..af646aa 100644 --- a/packages/vision-handoff/src/extension.ts +++ b/packages/vision-handoff/src/extension.ts @@ -16,6 +16,7 @@ import { readFile } from "node:fs/promises"; import { extname, isAbsolute, resolve as pathResolve } from "node:path"; +import { conversationStoreHandle } from "@dispatch/conversation-store"; import type { CredentialStore } from "@dispatch/credential-store"; import { credentialStoreHandle } from "@dispatch/credential-store"; import type { Extension, HostAPI, Manifest } from "@dispatch/kernel"; @@ -81,10 +82,6 @@ export async function activate(host: HostAPI): Promise { credentialStore, resolveModel, readFileAsDataUrl, - // Lazily resolve the session-orchestrator (for starting vision consultation - // turns). By the time consult_vision is called at runtime, all extensions - // have activated. The activated-manifests guard avoids a getService throw - // when the orchestrator isn't loaded. resolveOrchestrator: () => { const loaded = host.getExtensions().some((m) => m.id === "session-orchestrator"); if (!loaded) return undefined; @@ -94,6 +91,14 @@ export async function activate(host: HostAPI): Promise { return undefined; } }, + getImageTranscriptions: async (conversationId: string) => { + const store = host.getService(conversationStoreHandle); + return store.getImageTranscriptions(conversationId); + }, + setImageTranscription: async (conversationId: string, url: string, text: string) => { + const store = host.getService(conversationStoreHandle); + await store.setImageTranscription(conversationId, url, text); + }, logger: host.logger.child({ extensionId: "vision-handoff" }), }); diff --git a/packages/vision-handoff/src/pure.test.ts b/packages/vision-handoff/src/pure.test.ts index ea28288..e3fcf58 100644 --- a/packages/vision-handoff/src/pure.test.ts +++ b/packages/vision-handoff/src/pure.test.ts @@ -11,11 +11,15 @@ import { describe("isVisionCapable", () => { it("returns true when ModelInfo.vision is true", () => { - expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: true })).toBe(true); + expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: true })).toBe( + true, + ); }); it("returns false when ModelInfo.vision is false (overrides name heuristic)", () => { - expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: false })).toBe(false); + expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: false })).toBe( + false, + ); }); it("falls back to name heuristic when vision is absent (umans kimi + qwen)", () => { @@ -53,10 +57,7 @@ describe("findVisionModelName", () => { }); it("finds a vision model via ModelInfo.vision when name heuristic misses", async () => { - const name = await findVisionModelName( - ["umans/umans-glm-5.2", "umans/llama-vision"], - getInfo, - ); + const name = await findVisionModelName(["umans/umans-glm-5.2", "umans/llama-vision"], getInfo); expect(name).toBe("umans/llama-vision"); }); diff --git a/packages/vision-handoff/src/service.test.ts b/packages/vision-handoff/src/service.test.ts index dc54902..4667dbc 100644 --- a/packages/vision-handoff/src/service.test.ts +++ b/packages/vision-handoff/src/service.test.ts @@ -47,7 +47,8 @@ function makeDeps(overrides: Partial = {}): VisionHandoffDeps listCatalog: vi.fn(async () => catalog), getModelInfo: vi.fn(async (name: string) => infoMap[name]), resolve: vi.fn((name: string) => { - if (name === "umans/umans-kimi-k2.7") return { providerId: "umans", model: "umans-kimi-k2.7" }; + if (name === "umans/umans-kimi-k2.7") + return { providerId: "umans", model: "umans-kimi-k2.7" }; if (name === "umans/umans-glm-5.2") return { providerId: "umans", model: "umans-glm-5.2" }; return undefined; }), diff --git a/packages/vision-handoff/src/service.ts b/packages/vision-handoff/src/service.ts index 78f241f..7403c21 100644 --- a/packages/vision-handoff/src/service.ts +++ b/packages/vision-handoff/src/service.ts @@ -97,6 +97,24 @@ export interface VisionHandoffDeps { * with an error. Lazy so activation order doesn't matter. */ readonly resolveOrchestrator?: () => OrchestratorForVision | undefined; + /** + * Get the per-conversation cached image transcriptions (imageUrl → text). + * Used to avoid re-transcribing old images that were compacted to text on a + * previous turn. Optional — when absent, compaction still works but + * re-transcribes every turn (no caching). + */ + readonly getImageTranscriptions?: ( + conversationId: string, + ) => Promise>; + /** + * Upsert a single image transcription into the per-conversation cache. + * Optional — paired with getImageTranscriptions. + */ + readonly setImageTranscription?: ( + conversationId: string, + imageUrl: string, + transcription: string, + ) => Promise; /** Generate a new conversation ID for a consultation. Defaults to crypto.randomUUID. */ readonly generateId?: () => string; readonly logger?: Logger; @@ -128,6 +146,7 @@ export interface VisionHandoffService { currentModelName: string | undefined, opts?: { readonly conversationId?: string; + readonly imageLimit?: number; readonly signal?: AbortSignal; readonly logger?: Logger; }, @@ -198,6 +217,129 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando return { provider: resolved.provider, model: resolved.model, modelName: name }; } + /** + * Compact images for a vision-capable model: when the conversation has more + * image chunks than the limit, the oldest images are transcribed to text + * (one-time, cached in the conversation store) and stripped from the + * provider messages. Recent images (within the limit) stay native. + * + * The persisted history is NOT modified — only the provider's view. + * Transcriptions are cached so they're reused on subsequent turns (no + * re-transcription). When no caching deps are available, it still works but + * re-transcribes every turn. + */ + async function compactImagesForVisionModel( + messages: readonly ChatMessage[], + opts: + | { + readonly conversationId?: string; + readonly imageLimit?: number; + readonly signal?: AbortSignal; + readonly logger?: Logger; + } + | undefined, + currentModelName: string | undefined, + ): Promise { + void currentModelName; // reserved for future model-specific compaction logic + const limit = opts?.imageLimit; + // No limit or limit <= 0 → pass all images through (compaction disabled). + if (limit === undefined || limit <= 0) return messages; + + // Collect all image chunks in order (oldest first, across all messages). + const imageEntries: { msgIdx: number; chunkIdx: number; url: string }[] = []; + for (const [mi, msg] of messages.entries()) { + for (const [ci, chunk] of msg.chunks.entries()) { + if (chunk.type === "image") { + imageEntries.push({ msgIdx: mi, chunkIdx: ci, url: chunk.url }); + } + } + } + + // If within the limit, pass everything through natively. + if (imageEntries.length <= limit) return messages; + + // The oldest (imageEntries.length - limit) images need transcription. + const toTranscribeCount = imageEntries.length - limit; + const toTranscribe = imageEntries.slice(0, toTranscribeCount); + + // Load cached transcriptions. + const convId = opts?.conversationId; + const cache = + convId !== undefined && deps.getImageTranscriptions !== undefined + ? await deps.getImageTranscriptions(convId) + : new Map(); + + // Transcribe any that aren't cached yet (via the vision model). + const transcriptions = new Map(cache); + const vision = await resolveVisionModel(); + for (const entry of toTranscribe) { + if (transcriptions.has(entry.url)) continue; + if (vision === undefined) { + // No vision model available for transcription — use a placeholder. + transcriptions.set( + entry.url, + "[Image was compacted — no vision model available to transcribe it.]", + ); + continue; + } + try { + const prompt = + "Describe this image in detail. Include visible text (transcribe verbatim), " + + "key objects, layout, and notable details. This description will replace " + + "the image in a conversation history, so be thorough."; + const userMessage: ChatMessage = { + role: "user", + chunks: [ + { type: "text", text: prompt }, + { type: "image", url: entry.url }, + ], + }; + const stream = vision.provider.stream([userMessage], [], { + model: vision.model, + systemPrompt: "You are a vision assistant. Describe images faithfully and thoroughly.", + }); + const description = (await collectTextFromStream(stream)).trim(); + const text = + description.length > 0 ? description : "[Image transcription produced no output.]"; + transcriptions.set(entry.url, text); + // Cache it in the conversation store (if available). + if (convId !== undefined && deps.setImageTranscription !== undefined) { + await deps.setImageTranscription(convId, entry.url, text); + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + log?.warn("vision-handoff: image compaction transcription failed", { error: msg }); + transcriptions.set(entry.url, `[Image transcription failed: ${msg}]`); + } + } + + // Build the provider messages: replace transcribed images with text, + // keep recent images (within the limit) native. + const transcribedUrls = new Set(toTranscribe.map((e) => e.url)); + const result: ChatMessage[] = []; + for (const msg of messages) { + if (!msg.chunks.some((c) => c.type === "image")) { + result.push(msg); + continue; + } + const newChunks: Chunk[] = []; + for (const chunk of msg.chunks) { + if (chunk.type === "image" && transcribedUrls.has(chunk.url)) { + const transcription = transcriptions.get(chunk.url); + if (transcription !== undefined) { + newChunks.push({ type: "text", text: `[Compacted image]: ${transcription}` }); + } else { + newChunks.push(chunk); // fallback: keep the image + } + } else { + newChunks.push(chunk); + } + } + result.push({ role: msg.role, chunks: newChunks }); + } + return result; + } + const service: VisionHandoffService = { async isVisionCapable(modelName: string | undefined): Promise { if (modelName === undefined) return false; @@ -212,6 +354,7 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando currentModelName: string | undefined, opts?: { readonly conversationId?: string; + readonly imageLimit?: number; readonly signal?: AbortSignal; readonly logger?: Logger; }, @@ -219,13 +362,19 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando // Fast path: no images anywhere → nothing to do. if (!hasImageChunks(messages)) return messages; - // If the active model IS vision-capable, pass images through natively. - if (currentModelName !== undefined) { - const capable = await isVisionCapable(currentModelName, await getInfo(currentModelName)); - if (capable) return messages; + const isCapable = + currentModelName !== undefined && + (await isVisionCapable(currentModelName, await getInfo(currentModelName))); + + // ── Vision-capable model: image compaction ────────────────────────── + // When the conversation has more images than the limit, the oldest images + // are transcribed to text (one-time, cached) and stripped from the + // provider messages. Recent images (within the limit) stay native. + if (isCapable) { + return compactImagesForVisionModel(messages, opts, currentModelName); } - // Non-vision model: check if a vision model is available at all. + // ── Non-vision model: placeholders + consult_vision ────────────────── const vision = await resolveVisionModel(); const convId = opts?.conversationId; diff --git a/packages/vision-handoff/tsconfig.json b/packages/vision-handoff/tsconfig.json index ec597fc..b5439aa 100644 --- a/packages/vision-handoff/tsconfig.json +++ b/packages/vision-handoff/tsconfig.json @@ -5,6 +5,7 @@ "references": [ { "path": "../kernel" }, { "path": "../wire" }, + { "path": "../conversation-store" }, { "path": "../credential-store" }, { "path": "../openai-stream" } ] -- cgit v1.2.3