From d5633cf6e007eaf8255a44529a638d2466a74ba3 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 03:40:38 +0900 Subject: feat(vision-handoff): implement vision for capable models and universal vision handoff --- packages/transport-http/src/app.ts | 23 ++++++++--- packages/transport-http/src/logic.test.ts | 63 +++++++++++++++++++++++++++++++ packages/transport-http/src/logic.ts | 34 +++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) (limited to 'packages/transport-http/src') diff --git a/packages/transport-http/src/app.ts b/packages/transport-http/src/app.ts index 4fb295e..a9a23da 100644 --- a/packages/transport-http/src/app.ts +++ b/packages/transport-http/src/app.ts @@ -294,11 +294,14 @@ export function createApp(opts: CreateServerOptions): Hono { app.get("/models", async (c) => { try { const models = await opts.credentialStore.listCatalog(); - const modelInfo: Record = {}; + const modelInfo: Record = {}; for (const modelName of models) { const info = await opts.credentialStore.getModelInfo(modelName); - if (info?.contextWindow !== undefined) { - modelInfo[modelName] = { contextWindow: info.contextWindow }; + if (info?.contextWindow !== undefined || info?.vision === true) { + const entry: { contextWindow?: number; vision?: boolean } = {}; + if (info?.contextWindow !== undefined) entry.contextWindow = info.contextWindow; + if (info?.vision === true) entry.vision = true; + modelInfo[modelName] = entry; } } const body: ModelsResponse = { @@ -398,8 +401,16 @@ export function createApp(opts: CreateServerOptions): Hono { return c.json({ error: result.error }, 400); } - const { conversationId, message, model, cwd, computerId, reasoningEffort, workspaceId } = - result; + const { + conversationId, + message, + model, + cwd, + computerId, + reasoningEffort, + workspaceId, + images, + } = result; log.info("chat: request accepted", { conversationId, hasModel: model !== undefined, @@ -407,6 +418,7 @@ export function createApp(opts: CreateServerOptions): Hono { hasComputerId: computerId !== undefined, hasReasoningEffort: reasoningEffort !== undefined, hasWorkspaceId: workspaceId !== undefined, + imageCount: images?.length ?? 0, }); const events: AgentEvent[] = []; @@ -457,6 +469,7 @@ export function createApp(opts: CreateServerOptions): Hono { ...(computerId !== undefined ? { computerId } : {}), ...(reasoningEffort !== undefined ? { reasoningEffort } : {}), ...(workspaceId !== undefined ? { workspaceId } : {}), + ...(images !== undefined ? { images } : {}), }; opts.orchestrator diff --git a/packages/transport-http/src/logic.test.ts b/packages/transport-http/src/logic.test.ts index fc8302e..67632f3 100644 --- a/packages/transport-http/src/logic.test.ts +++ b/packages/transport-http/src/logic.test.ts @@ -182,6 +182,69 @@ describe("parseChatBody", () => { expect(result.reasoningEffort).toBeUndefined(); } }); + + // ── images ────────────────────────────────────────────────────────────── + + it("parses images array with data URLs", () => { + const result = parseChatBody( + { + message: "what is this?", + images: [ + { url: "data:image/png;base64,aaa" }, + { url: "data:image/jpeg;base64,bbb", mimeType: "image/jpeg" }, + ], + }, + fakeId, + ); + expect(isParseError(result)).toBe(false); + if (!isParseError(result)) { + expect(result.images).toHaveLength(2); + expect(result.images?.[0]?.url).toBe("data:image/png;base64,aaa"); + expect(result.images?.[1]?.mimeType).toBe("image/jpeg"); + } + }); + + it("parses images with http URLs", () => { + const result = parseChatBody( + { message: "hi", images: [{ url: "https://example.com/x.png" }] }, + fakeId, + ); + expect(isParseError(result)).toBe(false); + if (!isParseError(result)) { + expect(result.images?.[0]?.url).toBe("https://example.com/x.png"); + } + }); + + it("returns error when images is not an array", () => { + const result = parseChatBody({ message: "hi", images: "not-an-array" }, fakeId); + expect(isParseError(result)).toBe(true); + }); + + it("returns error when an image lacks a url", () => { + const result = parseChatBody({ message: "hi", images: [{ mimeType: "image/png" }] }, fakeId); + expect(isParseError(result)).toBe(true); + }); + + it("returns error when an image url is empty", () => { + const result = parseChatBody({ message: "hi", images: [{ url: "" }] }, fakeId); + expect(isParseError(result)).toBe(true); + }); + + it("omits images when absent (backward compatible)", () => { + const result = parseChatBody({ message: "hi" }, fakeId); + expect(isParseError(result)).toBe(false); + if (!isParseError(result)) { + expect(result.images).toBeUndefined(); + } + }); + + it("omits images when the array is empty", () => { + const result = parseChatBody({ message: "hi", images: [] }, fakeId); + expect(isParseError(result)).toBe(false); + if (!isParseError(result)) { + expect(result.images).toBeUndefined(); + } + }); }); describe("parseSinceSeq", () => { diff --git a/packages/transport-http/src/logic.ts b/packages/transport-http/src/logic.ts index 97ad426..a928147 100644 --- a/packages/transport-http/src/logic.ts +++ b/packages/transport-http/src/logic.ts @@ -55,6 +55,13 @@ export interface ChatCommand { readonly computerId?: string; readonly reasoningEffort?: ReasoningEffort; readonly workspaceId?: string; + /** + * Images attached to this turn (data URLs or http URLs). Parsed from the + * `ChatRequest.images` field; forwarded to the orchestrator which converts + * them to `image` chunks on the user message. Each entry must have a non-empty + * string `url`; `mimeType` is optional. + */ + readonly images?: readonly { readonly url: string; readonly mimeType?: string }[]; } export interface ParseError { @@ -121,6 +128,33 @@ export function parseChatBody(body: unknown, generateId: () => string): ParseRes (result as { workspaceId?: string }).workspaceId = obj.workspaceId; } + if (obj.images !== undefined) { + if (!Array.isArray(obj.images)) { + return { error: "Field 'images' must be an array" }; + } + const images: { url: string; mimeType?: string }[] = []; + for (const entry of obj.images) { + if (entry === null || typeof entry !== "object") { + return { error: "Each image must be an object with a 'url' string" }; + } + const img = entry as { url?: unknown; mimeType?: unknown }; + if (typeof img.url !== "string" || img.url.length === 0) { + return { error: "Each image must have a non-empty string 'url'" }; + } + const parsed: { url: string; mimeType?: string } = { url: img.url }; + if (img.mimeType !== undefined) { + if (typeof img.mimeType !== "string") { + return { error: "Field 'mimeType' on an image must be a string" }; + } + parsed.mimeType = img.mimeType; + } + images.push(parsed); + } + if (images.length > 0) { + (result as { images?: readonly { url: string; mimeType?: string }[] }).images = images; + } + } + return result; } -- cgit v1.2.3 From 2c91dc63802a386b1612ea0ed8c1e96b6f4421db Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 18:46:56 +0900 Subject: feat(vision): image compaction for vision-capable models + global vision settings --- bun.lock | 1 + packages/conversation-store/src/keys.ts | 8 ++ packages/conversation-store/src/store.ts | 77 +++++++++++ packages/session-orchestrator/src/orchestrator.ts | 3 + packages/transport-contract/src/index.ts | 17 +++ packages/transport-http/src/app.ts | 38 ++++++ packages/vision-handoff/package.json | 1 + packages/vision-handoff/src/extension.ts | 13 +- packages/vision-handoff/src/pure.test.ts | 13 +- packages/vision-handoff/src/service.test.ts | 3 +- packages/vision-handoff/src/service.ts | 159 +++++++++++++++++++++- packages/vision-handoff/tsconfig.json | 1 + 12 files changed, 318 insertions(+), 16 deletions(-) (limited to 'packages/transport-http/src') diff --git a/bun.lock b/bun.lock index 8a913d0..d9762f7 100644 --- a/bun.lock +++ b/bun.lock @@ -366,6 +366,7 @@ "name": "@dispatch/vision-handoff", "version": "0.0.0", "dependencies": { + "@dispatch/conversation-store": "workspace:*", "@dispatch/credential-store": "workspace:*", "@dispatch/kernel": "workspace:*", "@dispatch/openai-stream": "workspace:*", diff --git a/packages/conversation-store/src/keys.ts b/packages/conversation-store/src/keys.ts index b2c635d..6ec2bc5 100644 --- a/packages/conversation-store/src/keys.ts +++ b/packages/conversation-store/src/keys.ts @@ -66,6 +66,14 @@ export function compactThresholdKey(conversationId: string): string { return `conv:${conversationId}:compact-percent`; } +/** Per-conversation image transcription cache (JSON map of imageUrl → transcription). */ +export function imageTranscriptionsKey(conversationId: string): string { + return `conv:${conversationId}:image-transcriptions`; +} + +/** Global vision settings (image compaction limit + compaction model). */ +export const VISION_SETTINGS_KEY = "vision-settings"; + export function metaKey(conversationId: string): string { return `conv:${conversationId}:meta`; } diff --git a/packages/conversation-store/src/store.ts b/packages/conversation-store/src/store.ts index f90e809..69334e6 100644 --- a/packages/conversation-store/src/store.ts +++ b/packages/conversation-store/src/store.ts @@ -20,6 +20,7 @@ import { compactThresholdKey, computerKey, cwdKey, + imageTranscriptionsKey, metaKey, metricsKey, metricsPrefix, @@ -28,6 +29,7 @@ import { parseSeq, reasoningEffortKey, seqKey, + VISION_SETTINGS_KEY, workspaceKey, } from "./keys.js"; import { reconcileWithReport } from "./reconcile.js"; @@ -140,6 +142,35 @@ export interface ConversationStore { readonly getCompactPercent: (conversationId: string) => Promise; /** Set the compact percent (0-100, 0 = manual only). */ readonly setCompactPercent: (conversationId: string, percent: number) => Promise; + /** + * Get the per-conversation image transcription cache: a map of image URL → + * transcription text. Used by the vision handoff to avoid re-transcribing + * old images that were compacted to text on a previous turn. Returns an + * empty map when none are cached. + */ + readonly getImageTranscriptions: (conversationId: string) => Promise>; + /** + * Upsert a single image transcription into the per-conversation cache. + * Merges with any existing transcriptions (does NOT replace the whole map). + */ + readonly setImageTranscription: ( + conversationId: string, + imageUrl: string, + transcription: string, + ) => Promise; + /** + * Get the global vision settings (image compaction limit + compaction model). + * The limit defaults to 10 when never set; the compaction model defaults to + * null (auto-select). Shared across ALL conversations and vision models. + */ + readonly getVisionSettings: () => Promise<{ + readonly imageLimit: number; + readonly compactionModel: string | null; + }>; + /** Set the global vision image compaction limit (0 = disabled). */ + readonly setVisionImageLimit: (limit: number) => Promise; + /** Set the global vision compaction model (null = auto-select). */ + readonly setVisionCompactionModel: (model: string | null) => Promise; /** * Set the `compactedFrom` field on a conversation's metadata, pointing to * the archive conversation that holds the pre-compaction history. @@ -1004,6 +1035,52 @@ export function createConversationStore( } }, + async getImageTranscriptions(conversationId) { + const raw = await storage.get(imageTranscriptionsKey(conversationId)); + if (raw === null) return new Map(); + try { + const obj = JSON.parse(raw) as Record; + return new Map(Object.entries(obj)); + } catch { + return new Map(); + } + }, + + async setImageTranscription(conversationId, imageUrl, transcription) { + const existing = await this.getImageTranscriptions(conversationId); + const merged = new Map(existing); + merged.set(imageUrl, transcription); + const obj: Record = {}; + for (const [k, v] of merged) obj[k] = v; + await storage.set(imageTranscriptionsKey(conversationId), JSON.stringify(obj)); + }, + + async getVisionSettings() { + const raw = await storage.get(VISION_SETTINGS_KEY); + if (raw === null) return { imageLimit: 10, compactionModel: null }; + try { + const obj = JSON.parse(raw) as { imageLimit?: number; compactionModel?: string | null }; + return { + imageLimit: typeof obj.imageLimit === "number" ? obj.imageLimit : 10, + compactionModel: obj.compactionModel ?? null, + }; + } catch { + return { imageLimit: 10, compactionModel: null }; + } + }, + + async setVisionImageLimit(limit) { + const current = await this.getVisionSettings(); + const obj = { imageLimit: limit, compactionModel: current.compactionModel }; + await storage.set(VISION_SETTINGS_KEY, JSON.stringify(obj)); + }, + + async setVisionCompactionModel(model) { + const current = await this.getVisionSettings(); + const obj = { imageLimit: current.imageLimit, compactionModel: model }; + await storage.set(VISION_SETTINGS_KEY, JSON.stringify(obj)); + }, + async setCompactedFrom(conversationId, newConversationId) { const raw = await storage.get(metaKey(conversationId)); const existing = raw !== null ? parseMetaRow(raw) : null; diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts index 4f4bb3e..c0493f3 100644 --- a/packages/session-orchestrator/src/orchestrator.ts +++ b/packages/session-orchestrator/src/orchestrator.ts @@ -54,6 +54,7 @@ export interface VisionHandoffService { currentModelName: string | undefined, opts?: { readonly conversationId?: string; + readonly imageLimit?: number; readonly signal?: AbortSignal; readonly logger?: Logger; }, @@ -776,11 +777,13 @@ export function createSessionOrchestrator( const visionHandoff = deps.resolveVisionHandoff?.(); let providerMessages: readonly ChatMessage[] = [...history, userMsg]; if (visionHandoff !== undefined) { + const visionSettings = await deps.conversationStore.getVisionSettings(); providerMessages = await visionHandoff.prepareForProvider( providerMessages, effectiveModelName, { conversationId, + imageLimit: visionSettings.imageLimit, signal: controller.signal, ...(turnLogger !== undefined ? { logger: turnLogger } : {}), }, diff --git a/packages/transport-contract/src/index.ts b/packages/transport-contract/src/index.ts index 0444f29..94897f7 100644 --- a/packages/transport-contract/src/index.ts +++ b/packages/transport-contract/src/index.ts @@ -411,6 +411,23 @@ export interface SystemPromptVariablesResponse { readonly variables: readonly SystemPromptVariable[]; } +// ─── Vision settings (global) ────────────────────────────────────────────────── + +/** + * Response of `GET /settings/vision` — the global vision configuration shared + * across all conversations and vision models. + */ +export interface VisionSettingsResponse { + readonly imageLimit: number; + readonly compactionModel: string | null; +} + +/** Body of `PUT /settings/vision` — a partial update. */ +export interface SetVisionSettingsRequest { + readonly imageLimit?: number; + readonly compactionModel?: string | null; +} + // ─── Message queue (steering) ───────────────────────────────────────────────── /** diff --git a/packages/transport-http/src/app.ts b/packages/transport-http/src/app.ts index a9a23da..ea216e1 100644 --- a/packages/transport-http/src/app.ts +++ b/packages/transport-http/src/app.ts @@ -38,6 +38,7 @@ import type { ThroughputResponse, TitleResponse, UpdateHeartbeatRequest, + VisionSettingsResponse, WarmResponse, WorkspaceListResponse, WorkspaceResponse, @@ -1594,6 +1595,43 @@ export function createApp(opts: CreateServerOptions): Hono { return c.json(response, 200); }); + app.get("/settings/vision", async (c) => { + const settings = await opts.conversationStore.getVisionSettings(); + const body: VisionSettingsResponse = settings; + return c.json(body, 200); + }); + + app.put("/settings/vision", async (c) => { + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid JSON body" }, 400); + } + const obj = body as { imageLimit?: unknown; compactionModel?: unknown }; + if (obj.imageLimit !== undefined) { + if ( + typeof obj.imageLimit !== "number" || + !Number.isInteger(obj.imageLimit) || + obj.imageLimit < 0 + ) { + return c.json({ error: "imageLimit must be a non-negative integer" }, 400); + } + await opts.conversationStore.setVisionImageLimit(obj.imageLimit); + log.info("vision: image limit set", { imageLimit: obj.imageLimit }); + } + if (obj.compactionModel !== undefined) { + if (obj.compactionModel !== null && typeof obj.compactionModel !== "string") { + return c.json({ error: "compactionModel must be a string or null" }, 400); + } + await opts.conversationStore.setVisionCompactionModel(obj.compactionModel); + log.info("vision: compaction model set", { compactionModel: obj.compactionModel }); + } + const settings = await opts.conversationStore.getVisionSettings(); + const response: VisionSettingsResponse = settings; + return c.json(response, 200); + }); + // ─── Static frontend serving (catch-all, API routes take precedence) ────── if (opts.webDir !== undefined) { const webDir = opts.webDir; diff --git a/packages/vision-handoff/package.json b/packages/vision-handoff/package.json index a88ab49..b11f7ee 100644 --- a/packages/vision-handoff/package.json +++ b/packages/vision-handoff/package.json @@ -6,6 +6,7 @@ "main": "dist/index.js", "types": "dist/index.d.ts", "dependencies": { + "@dispatch/conversation-store": "workspace:*", "@dispatch/credential-store": "workspace:*", "@dispatch/kernel": "workspace:*", "@dispatch/openai-stream": "workspace:*" diff --git a/packages/vision-handoff/src/extension.ts b/packages/vision-handoff/src/extension.ts index 2f75a6b..af646aa 100644 --- a/packages/vision-handoff/src/extension.ts +++ b/packages/vision-handoff/src/extension.ts @@ -16,6 +16,7 @@ import { readFile } from "node:fs/promises"; import { extname, isAbsolute, resolve as pathResolve } from "node:path"; +import { conversationStoreHandle } from "@dispatch/conversation-store"; import type { CredentialStore } from "@dispatch/credential-store"; import { credentialStoreHandle } from "@dispatch/credential-store"; import type { Extension, HostAPI, Manifest } from "@dispatch/kernel"; @@ -81,10 +82,6 @@ export async function activate(host: HostAPI): Promise { credentialStore, resolveModel, readFileAsDataUrl, - // Lazily resolve the session-orchestrator (for starting vision consultation - // turns). By the time consult_vision is called at runtime, all extensions - // have activated. The activated-manifests guard avoids a getService throw - // when the orchestrator isn't loaded. resolveOrchestrator: () => { const loaded = host.getExtensions().some((m) => m.id === "session-orchestrator"); if (!loaded) return undefined; @@ -94,6 +91,14 @@ export async function activate(host: HostAPI): Promise { return undefined; } }, + getImageTranscriptions: async (conversationId: string) => { + const store = host.getService(conversationStoreHandle); + return store.getImageTranscriptions(conversationId); + }, + setImageTranscription: async (conversationId: string, url: string, text: string) => { + const store = host.getService(conversationStoreHandle); + await store.setImageTranscription(conversationId, url, text); + }, logger: host.logger.child({ extensionId: "vision-handoff" }), }); diff --git a/packages/vision-handoff/src/pure.test.ts b/packages/vision-handoff/src/pure.test.ts index ea28288..e3fcf58 100644 --- a/packages/vision-handoff/src/pure.test.ts +++ b/packages/vision-handoff/src/pure.test.ts @@ -11,11 +11,15 @@ import { describe("isVisionCapable", () => { it("returns true when ModelInfo.vision is true", () => { - expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: true })).toBe(true); + expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: true })).toBe( + true, + ); }); it("returns false when ModelInfo.vision is false (overrides name heuristic)", () => { - expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: false })).toBe(false); + expect(isVisionCapable("umans/umans-kimi-k2.7", { id: "umans-kimi-k2.7", vision: false })).toBe( + false, + ); }); it("falls back to name heuristic when vision is absent (umans kimi + qwen)", () => { @@ -53,10 +57,7 @@ describe("findVisionModelName", () => { }); it("finds a vision model via ModelInfo.vision when name heuristic misses", async () => { - const name = await findVisionModelName( - ["umans/umans-glm-5.2", "umans/llama-vision"], - getInfo, - ); + const name = await findVisionModelName(["umans/umans-glm-5.2", "umans/llama-vision"], getInfo); expect(name).toBe("umans/llama-vision"); }); diff --git a/packages/vision-handoff/src/service.test.ts b/packages/vision-handoff/src/service.test.ts index dc54902..4667dbc 100644 --- a/packages/vision-handoff/src/service.test.ts +++ b/packages/vision-handoff/src/service.test.ts @@ -47,7 +47,8 @@ function makeDeps(overrides: Partial = {}): VisionHandoffDeps listCatalog: vi.fn(async () => catalog), getModelInfo: vi.fn(async (name: string) => infoMap[name]), resolve: vi.fn((name: string) => { - if (name === "umans/umans-kimi-k2.7") return { providerId: "umans", model: "umans-kimi-k2.7" }; + if (name === "umans/umans-kimi-k2.7") + return { providerId: "umans", model: "umans-kimi-k2.7" }; if (name === "umans/umans-glm-5.2") return { providerId: "umans", model: "umans-glm-5.2" }; return undefined; }), diff --git a/packages/vision-handoff/src/service.ts b/packages/vision-handoff/src/service.ts index 78f241f..7403c21 100644 --- a/packages/vision-handoff/src/service.ts +++ b/packages/vision-handoff/src/service.ts @@ -97,6 +97,24 @@ export interface VisionHandoffDeps { * with an error. Lazy so activation order doesn't matter. */ readonly resolveOrchestrator?: () => OrchestratorForVision | undefined; + /** + * Get the per-conversation cached image transcriptions (imageUrl → text). + * Used to avoid re-transcribing old images that were compacted to text on a + * previous turn. Optional — when absent, compaction still works but + * re-transcribes every turn (no caching). + */ + readonly getImageTranscriptions?: ( + conversationId: string, + ) => Promise>; + /** + * Upsert a single image transcription into the per-conversation cache. + * Optional — paired with getImageTranscriptions. + */ + readonly setImageTranscription?: ( + conversationId: string, + imageUrl: string, + transcription: string, + ) => Promise; /** Generate a new conversation ID for a consultation. Defaults to crypto.randomUUID. */ readonly generateId?: () => string; readonly logger?: Logger; @@ -128,6 +146,7 @@ export interface VisionHandoffService { currentModelName: string | undefined, opts?: { readonly conversationId?: string; + readonly imageLimit?: number; readonly signal?: AbortSignal; readonly logger?: Logger; }, @@ -198,6 +217,129 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando return { provider: resolved.provider, model: resolved.model, modelName: name }; } + /** + * Compact images for a vision-capable model: when the conversation has more + * image chunks than the limit, the oldest images are transcribed to text + * (one-time, cached in the conversation store) and stripped from the + * provider messages. Recent images (within the limit) stay native. + * + * The persisted history is NOT modified — only the provider's view. + * Transcriptions are cached so they're reused on subsequent turns (no + * re-transcription). When no caching deps are available, it still works but + * re-transcribes every turn. + */ + async function compactImagesForVisionModel( + messages: readonly ChatMessage[], + opts: + | { + readonly conversationId?: string; + readonly imageLimit?: number; + readonly signal?: AbortSignal; + readonly logger?: Logger; + } + | undefined, + currentModelName: string | undefined, + ): Promise { + void currentModelName; // reserved for future model-specific compaction logic + const limit = opts?.imageLimit; + // No limit or limit <= 0 → pass all images through (compaction disabled). + if (limit === undefined || limit <= 0) return messages; + + // Collect all image chunks in order (oldest first, across all messages). + const imageEntries: { msgIdx: number; chunkIdx: number; url: string }[] = []; + for (const [mi, msg] of messages.entries()) { + for (const [ci, chunk] of msg.chunks.entries()) { + if (chunk.type === "image") { + imageEntries.push({ msgIdx: mi, chunkIdx: ci, url: chunk.url }); + } + } + } + + // If within the limit, pass everything through natively. + if (imageEntries.length <= limit) return messages; + + // The oldest (imageEntries.length - limit) images need transcription. + const toTranscribeCount = imageEntries.length - limit; + const toTranscribe = imageEntries.slice(0, toTranscribeCount); + + // Load cached transcriptions. + const convId = opts?.conversationId; + const cache = + convId !== undefined && deps.getImageTranscriptions !== undefined + ? await deps.getImageTranscriptions(convId) + : new Map(); + + // Transcribe any that aren't cached yet (via the vision model). + const transcriptions = new Map(cache); + const vision = await resolveVisionModel(); + for (const entry of toTranscribe) { + if (transcriptions.has(entry.url)) continue; + if (vision === undefined) { + // No vision model available for transcription — use a placeholder. + transcriptions.set( + entry.url, + "[Image was compacted — no vision model available to transcribe it.]", + ); + continue; + } + try { + const prompt = + "Describe this image in detail. Include visible text (transcribe verbatim), " + + "key objects, layout, and notable details. This description will replace " + + "the image in a conversation history, so be thorough."; + const userMessage: ChatMessage = { + role: "user", + chunks: [ + { type: "text", text: prompt }, + { type: "image", url: entry.url }, + ], + }; + const stream = vision.provider.stream([userMessage], [], { + model: vision.model, + systemPrompt: "You are a vision assistant. Describe images faithfully and thoroughly.", + }); + const description = (await collectTextFromStream(stream)).trim(); + const text = + description.length > 0 ? description : "[Image transcription produced no output.]"; + transcriptions.set(entry.url, text); + // Cache it in the conversation store (if available). + if (convId !== undefined && deps.setImageTranscription !== undefined) { + await deps.setImageTranscription(convId, entry.url, text); + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + log?.warn("vision-handoff: image compaction transcription failed", { error: msg }); + transcriptions.set(entry.url, `[Image transcription failed: ${msg}]`); + } + } + + // Build the provider messages: replace transcribed images with text, + // keep recent images (within the limit) native. + const transcribedUrls = new Set(toTranscribe.map((e) => e.url)); + const result: ChatMessage[] = []; + for (const msg of messages) { + if (!msg.chunks.some((c) => c.type === "image")) { + result.push(msg); + continue; + } + const newChunks: Chunk[] = []; + for (const chunk of msg.chunks) { + if (chunk.type === "image" && transcribedUrls.has(chunk.url)) { + const transcription = transcriptions.get(chunk.url); + if (transcription !== undefined) { + newChunks.push({ type: "text", text: `[Compacted image]: ${transcription}` }); + } else { + newChunks.push(chunk); // fallback: keep the image + } + } else { + newChunks.push(chunk); + } + } + result.push({ role: msg.role, chunks: newChunks }); + } + return result; + } + const service: VisionHandoffService = { async isVisionCapable(modelName: string | undefined): Promise { if (modelName === undefined) return false; @@ -212,6 +354,7 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando currentModelName: string | undefined, opts?: { readonly conversationId?: string; + readonly imageLimit?: number; readonly signal?: AbortSignal; readonly logger?: Logger; }, @@ -219,13 +362,19 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando // Fast path: no images anywhere → nothing to do. if (!hasImageChunks(messages)) return messages; - // If the active model IS vision-capable, pass images through natively. - if (currentModelName !== undefined) { - const capable = await isVisionCapable(currentModelName, await getInfo(currentModelName)); - if (capable) return messages; + const isCapable = + currentModelName !== undefined && + (await isVisionCapable(currentModelName, await getInfo(currentModelName))); + + // ── Vision-capable model: image compaction ────────────────────────── + // When the conversation has more images than the limit, the oldest images + // are transcribed to text (one-time, cached) and stripped from the + // provider messages. Recent images (within the limit) stay native. + if (isCapable) { + return compactImagesForVisionModel(messages, opts, currentModelName); } - // Non-vision model: check if a vision model is available at all. + // ── Non-vision model: placeholders + consult_vision ────────────────── const vision = await resolveVisionModel(); const convId = opts?.conversationId; diff --git a/packages/vision-handoff/tsconfig.json b/packages/vision-handoff/tsconfig.json index ec597fc..b5439aa 100644 --- a/packages/vision-handoff/tsconfig.json +++ b/packages/vision-handoff/tsconfig.json @@ -5,6 +5,7 @@ "references": [ { "path": "../kernel" }, { "path": "../wire" }, + { "path": "../conversation-store" }, { "path": "../credential-store" }, { "path": "../openai-stream" } ] -- cgit v1.2.3 From 2e741d1c1ac309327aff4fed0e248bc5baa342d4 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 20:06:47 +0900 Subject: feat(vision): store images in tmp dir instead of SQLite — compact URLs + purge on compaction/close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/session-orchestrator/src/orchestrator.ts | 30 ++++- packages/transport-http/src/app.ts | 31 +++++ packages/vision-handoff/src/extension.ts | 88 +++++++++++++- packages/vision-handoff/src/service.ts | 135 +++++++++++++++++++++- 4 files changed, 279 insertions(+), 5 deletions(-) (limited to 'packages/transport-http/src') diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts index c0493f3..045b88d 100644 --- a/packages/session-orchestrator/src/orchestrator.ts +++ b/packages/session-orchestrator/src/orchestrator.ts @@ -49,6 +49,20 @@ import type { ToolAssembly } from "./tools-filter.js"; * call `consult_vision`) and the images are registered for tool access. */ export interface VisionHandoffService { + /** + * Store images to tmp files and return compact URLs. Each input image's data + * URL is saved to a tmp file and replaced with a compact HTTP path so the + * persisted conversation store holds a tiny string, not megabytes of base64. + * When `saveImageToTmp` is not configured, data URLs pass through unchanged. + */ + readonly storeImages: ( + conversationId: string, + images: readonly ImageInput[], + ) => Promise; + + /** Delete all tmp images for a conversation (on close). Best-effort. */ + readonly purgeConversationImages: (conversationId: string) => Promise; + readonly prepareForProvider: ( messages: readonly ChatMessage[], currentModelName: string | undefined, @@ -625,7 +639,18 @@ export function createSessionOrchestrator( const effectiveModelName = resolveModelName(modelName, storedModel); const history = await deps.conversationStore.load(conversationId); - const userMsg = buildUserMessage(text, images); + + // Store images to tmp files (compact URLs) BEFORE building the user + // message so the persisted chunks hold tiny URL references, not + // megabytes of base64 data URLs. When the vision-handoff service isn't + // loaded, images pass through unchanged (backward compatible). + const visionHandoffForStore = deps.resolveVisionHandoff?.(); + const storedImages = + visionHandoffForStore !== undefined && images !== undefined + ? await visionHandoffForStore.storeImages(conversationId, images) + : images; + + const userMsg = buildUserMessage(text, storedImages); // Workspace assignment for new conversations happens BEFORE // effective-cwd resolution (see workspaceSetupPromise above) so @@ -988,6 +1013,9 @@ export function createSessionOrchestrator( }); }); void deps.conversationStore.setConversationStatus(conversationId, "closed"); + // Purge tmp images for this conversation (best-effort, fire-and-forget). + const vh = deps.resolveVisionHandoff?.(); + if (vh !== undefined) void vh.purgeConversationImages(conversationId); return { abortedTurn }; }, diff --git a/packages/transport-http/src/app.ts b/packages/transport-http/src/app.ts index ea216e1..16c4167 100644 --- a/packages/transport-http/src/app.ts +++ b/packages/transport-http/src/app.ts @@ -201,6 +201,37 @@ export function createApp(opts: CreateServerOptions): Hono { app.get("/health", (c) => c.json({ ok: true })); + // ── Tmp image serving (vision handoff) ────────────────────────────────────── + app.get("/images/:conversationId/:imageId", async (c) => { + const conversationId = c.req.param("conversationId"); + const imageId = c.req.param("imageId"); + if (imageId.includes("/") || imageId.includes("..")) { + return c.json({ error: "Invalid image ID" }, 400); + } + const imageDir = process.env.DISPATCH_IMAGE_DIR ?? "/tmp/dispatch/images"; + const { join } = await import("node:path"); + const { readFile: fsReadFile } = await import("node:fs/promises"); + const filePath = join(imageDir, conversationId, imageId); + try { + const buf = await fsReadFile(filePath); + const ext = imageId.toLowerCase(); + const mime = ext.endsWith(".png") + ? "image/png" + : ext.endsWith(".jpg") || ext.endsWith(".jpeg") + ? "image/jpeg" + : ext.endsWith(".webp") + ? "image/webp" + : ext.endsWith(".gif") + ? "image/gif" + : ext.endsWith(".bmp") + ? "image/bmp" + : "application/octet-stream"; + return new Response(buf, { headers: { "Content-Type": mime, "Cache-Control": "no-cache" } }); + } catch { + return c.json({ error: "Image not found" }, 404); + } + }); + app.get("/conversations/:id/metrics", async (c) => { const conversationId = c.req.param("id"); diff --git a/packages/vision-handoff/src/extension.ts b/packages/vision-handoff/src/extension.ts index af646aa..faf4621 100644 --- a/packages/vision-handoff/src/extension.ts +++ b/packages/vision-handoff/src/extension.ts @@ -9,13 +9,18 @@ * image + the model's specific question, and returns the conversation ID + the * vision model's answer. Follow-ups go through the dispatch CLI. * + * Images are saved to a tmp directory (`/tmp/dispatch/images//`) so the + * conversation store (SQLite) only holds a compact URL reference — not + * megabytes of base64. Tmp files are purged on reboot (ephemeral dir), after + * compaction (the transcription replaces the image), and on conversation close. + * * Effects (filesystem, orchestrator) live here in the shell, injected into the * service. The pure decisions live in `pure.ts`. No `console.*`; logging via * `host.logger`. */ -import { readFile } from "node:fs/promises"; -import { extname, isAbsolute, resolve as pathResolve } from "node:path"; +import { mkdir, readFile, rm, unlink, writeFile } from "node:fs/promises"; +import { extname, isAbsolute, join, resolve as pathResolve } from "node:path"; import { conversationStoreHandle } from "@dispatch/conversation-store"; import type { CredentialStore } from "@dispatch/credential-store"; import { credentialStoreHandle } from "@dispatch/credential-store"; @@ -38,6 +43,8 @@ export const manifest: Manifest = { contributes: { services: ["vision-handoff/service"], tools: ["consult_vision"] }, }; +const IMAGE_DIR = process.env.DISPATCH_IMAGE_DIR ?? "/tmp/dispatch/images"; + /** MIME types for recognized image extensions. */ const MIME_BY_EXT: Readonly> = { ".png": "image/png", @@ -48,6 +55,15 @@ const MIME_BY_EXT: Readonly> = { ".bmp": "image/bmp", }; +/** Reverse: MIME → extension. */ +const EXT_BY_MIME: Readonly> = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/webp": ".webp", + "image/gif": ".gif", + "image/bmp": ".bmp", +}; + /** * Read an image file from disk as a base64 data URL. Resolves relative paths * against the cwd (the conversation's working directory). Throws on missing @@ -61,6 +77,70 @@ async function readFileAsDataUrl(path: string, cwd?: string): Promise { return `data:${mime};base64,${buf.toString("base64")}`; } +/** + * Save a data URL image to a tmp file and return a compact HTTP path. + * The compact URL (`/images//.`) is what gets + * persisted in the conversation store — a tiny string, not megabytes of base64. + */ +async function saveImageToTmp( + conversationId: string, + dataUrl: string, + mimeType?: string, +): Promise { + const mime = mimeType ?? "image/png"; + const ext = EXT_BY_MIME[mime] ?? ".png"; + const imageId = `${crypto.randomUUID()}${ext}`; + const dir = join(IMAGE_DIR, conversationId); + await mkdir(dir, { recursive: true }); + const filePath = join(dir, imageId); + const base64 = dataUrl.split(",")[1] ?? ""; + await writeFile(filePath, Buffer.from(base64, "base64")); + return `/images/${conversationId}/${imageId}`; +} + +/** + * Resolve a compact URL (`/images//`) back to a data URL by + * reading the tmp file. Data URLs and HTTP URLs pass through unchanged. + */ +async function resolveImageUrl(url: string): Promise { + if (url.startsWith("data:") || url.startsWith("http")) return url; + if (!url.startsWith("/images/")) return url; + const parts = url.split("/"); // ["", "images", convId, imageId] + const convId = parts[2]; + const imageId = parts[3]; + if (convId === undefined || imageId === undefined) return url; + const filePath = join(IMAGE_DIR, convId, imageId); + const buf = await readFile(filePath); + const ext = extname(imageId).toLowerCase(); + const mime = MIME_BY_EXT[ext] ?? "image/png"; + return `data:${mime};base64,${buf.toString("base64")}`; +} + +/** Delete a single tmp image file (after compaction — best-effort). */ +async function deleteTmpImage(compactUrl: string): Promise { + if (!compactUrl.startsWith("/images/")) return; + const parts = compactUrl.split("/"); + const convId = parts[2]; + const imageId = parts[3]; + if (convId === undefined || imageId === undefined) return; + const filePath = join(IMAGE_DIR, convId, imageId); + try { + await unlink(filePath); + } catch { + // Best-effort — file may already be deleted. + } +} + +/** Delete all tmp images for a conversation (on close — best-effort). */ +async function deleteConversationImages(conversationId: string): Promise { + const dir = join(IMAGE_DIR, conversationId); + try { + await rm(dir, { recursive: true, force: true }); + } catch { + // Best-effort. + } +} + export async function activate(host: HostAPI): Promise { const credentialStore = host.getService(credentialStoreHandle) as CredentialStore | undefined; if (credentialStore === undefined) { @@ -82,6 +162,10 @@ export async function activate(host: HostAPI): Promise { credentialStore, resolveModel, readFileAsDataUrl, + saveImageToTmp, + resolveImageUrl, + deleteTmpImage, + deleteConversationImages, resolveOrchestrator: () => { const loaded = host.getExtensions().some((m) => m.id === "session-orchestrator"); if (!loaded) return undefined; diff --git a/packages/vision-handoff/src/service.ts b/packages/vision-handoff/src/service.ts index 7403c21..cc13d93 100644 --- a/packages/vision-handoff/src/service.ts +++ b/packages/vision-handoff/src/service.ts @@ -115,6 +115,37 @@ export interface VisionHandoffDeps { imageUrl: string, transcription: string, ) => Promise; + /** + * Save an image data URL to a tmp file and return a compact URL + * (`/images//.`) that can be persisted in the + * conversation store instead of the full data URL (which would be megabytes). + * The frontend serves the image via `GET /images/...`; the provider resolves + * it back to a data URL via {@link resolveImageUrl} at runtime. When `undefined`, + * data URLs pass through unchanged (images persist in SQLite — the large-DB + * path, for environments without tmp file support). + */ + readonly saveImageToTmp?: ( + conversationId: string, + dataUrl: string, + mimeType?: string, + ) => Promise; + /** + * Resolve a compact URL (`/images/...`) back to a data URL by reading the tmp + * file. Data URLs and HTTP URLs pass through unchanged. Paired with + * {@link saveImageToTmp}. + */ + readonly resolveImageUrl?: (url: string) => Promise; + /** + * Delete a tmp image file (after it has been compacted to text — the + * transcription is cached, the raw image is no longer needed). Best-effort: + * errors are logged, not thrown. + */ + readonly deleteTmpImage?: (compactUrl: string) => Promise; + /** + * Delete all tmp images for a conversation (on conversation close). + * Best-effort. + */ + readonly deleteConversationImages?: (conversationId: string) => Promise; /** Generate a new conversation ID for a consultation. Defaults to crypto.randomUUID. */ readonly generateId?: () => string; readonly logger?: Logger; @@ -127,6 +158,24 @@ export interface VisionHandoffService { */ readonly isVisionCapable: (modelName: string | undefined) => Promise; + /** + * Store images to tmp files and return compact URLs. Each input image's data + * URL is saved to `/tmp/dispatch/images//.` and + * replaced with a compact HTTP path (`/images//.`) + * so the persisted conversation store holds a tiny string, not megabytes of + * base64. When `saveImageToTmp` is not configured, data URLs pass through + * unchanged (backward compatible). + */ + readonly storeImages: ( + conversationId: string, + images: readonly ImageInput[], + ) => Promise; + + /** + * Delete all tmp images for a conversation (on close). Best-effort. + */ + readonly purgeConversationImages: (conversationId: string) => Promise; + /** * Resolve a vision-capable model from the catalog (any provider). Returns * `undefined` when none is available. @@ -306,6 +355,15 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando if (convId !== undefined && deps.setImageTranscription !== undefined) { await deps.setImageTranscription(convId, entry.url, text); } + // The image has been transcribed to text — delete the tmp file + // (the transcription is cached, the raw image is no longer needed). + if (deps.deleteTmpImage !== undefined) { + try { + await deps.deleteTmpImage(entry.url); + } catch { + // Best-effort — don't let cleanup failure break the turn. + } + } } catch (err) { const msg = err instanceof Error ? err.message : String(err); log?.warn("vision-handoff: image compaction transcription failed", { error: msg }); @@ -340,6 +398,42 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando return result; } + async function resolveImageUrlsInMessages( + messages: readonly ChatMessage[], + ): Promise { + if (deps.resolveImageUrl === undefined) return messages; + let hasCompact = false; + for (const msg of messages) { + if (msg.chunks.some((c) => c.type === "image")) { + hasCompact = true; + break; + } + } + if (!hasCompact) return messages; + const result: ChatMessage[] = []; + for (const msg of messages) { + if (!msg.chunks.some((c) => c.type === "image")) { + result.push(msg); + continue; + } + const newChunks: Chunk[] = []; + for (const chunk of msg.chunks) { + if (chunk.type === "image") { + const dataUrl = await deps.resolveImageUrl!(chunk.url); + newChunks.push({ + type: "image", + url: dataUrl, + ...(chunk.mimeType !== undefined ? { mimeType: chunk.mimeType } : {}), + }); + } else { + newChunks.push(chunk); + } + } + result.push({ role: msg.role, chunks: newChunks }); + } + return result; + } + const service: VisionHandoffService = { async isVisionCapable(modelName: string | undefined): Promise { if (modelName === undefined) return false; @@ -347,6 +441,38 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando return isVisionCapable(modelName, info); }, + async storeImages( + conversationId: string, + images: readonly ImageInput[], + ): Promise { + if (deps.saveImageToTmp === undefined) return images; + const result: ImageInput[] = []; + for (const img of images) { + if (img.url.startsWith("data:")) { + const compactUrl = await deps.saveImageToTmp(conversationId, img.url, img.mimeType); + result.push({ + url: compactUrl, + ...(img.mimeType !== undefined ? { mimeType: img.mimeType } : {}), + }); + } else { + result.push(img); + } + } + return result; + }, + + async purgeConversationImages(conversationId: string): Promise { + if (deps.deleteConversationImages === undefined) return; + try { + await deps.deleteConversationImages(conversationId); + } catch (err) { + log?.warn("vision-handoff: failed to purge conversation images", { + conversationId, + error: err instanceof Error ? err.message : String(err), + }); + } + }, + resolveVisionModel, async prepareForProvider( @@ -362,6 +488,11 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando // Fast path: no images anywhere → nothing to do. if (!hasImageChunks(messages)) return messages; + // Resolve compact URLs (/images/...) → data URLs for the provider. + // The persisted chunks store compact URLs (tiny strings); the provider + // needs data URLs (read from tmp files at runtime). + const resolved = await resolveImageUrlsInMessages(messages); + const isCapable = currentModelName !== undefined && (await isVisionCapable(currentModelName, await getInfo(currentModelName))); @@ -371,7 +502,7 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando // are transcribed to text (one-time, cached) and stripped from the // provider messages. Recent images (within the limit) stay native. if (isCapable) { - return compactImagesForVisionModel(messages, opts, currentModelName); + return compactImagesForVisionModel(resolved, opts, currentModelName); } // ── Non-vision model: placeholders + consult_vision ────────────────── @@ -388,7 +519,7 @@ export function createVisionHandoffService(deps: VisionHandoffDeps): VisionHando // per-conversation registry so the consult_vision tool can look it up. let seqId = 0; const result: ChatMessage[] = []; - for (const msg of messages) { + for (const msg of resolved) { if (!msg.chunks.some((c) => c.type === "image")) { result.push(msg); continue; -- cgit v1.2.3