diff options
Diffstat (limited to 'src/app')
| -rw-r--r-- | src/app/App.svelte | 55 | ||||
| -rw-r--r-- | src/app/store.svelte.ts | 103 | ||||
| -rw-r--r-- | src/app/store.test.ts | 152 |
3 files changed, 300 insertions, 10 deletions
diff --git a/src/app/App.svelte b/src/app/App.svelte index b46885c..59f949c 100644 --- a/src/app/App.svelte +++ b/src/app/App.svelte @@ -1,5 +1,5 @@ <script lang="ts"> - import type { ReasoningEffort } from "@dispatch/transport-contract"; + import type { ImageInput, ReasoningEffort } from "@dispatch/transport-contract"; import type { InvokeMessage } from "@dispatch/ui-contract"; import { tick } from "svelte"; import Table from "../components/Table.svelte"; @@ -86,6 +86,13 @@ type SaveSystemPrompt as SaveSystemPromptAlias, manifest as systemPromptManifest, } from "../features/system-prompt"; + import { + VisionSettingsView, + manifest as visionManifest, + type LoadVisionSettingsResult, + type SaveVisionSettingsResult, + type VisionSettingsPatch, + } from "../features/vision"; import type { AppStore } from "./store.svelte"; import ErrorModal from "./ErrorModal.svelte"; import { createLocalStore } from "../adapters/local-storage"; @@ -156,6 +163,7 @@ systemPromptManifest, heartbeatManifest, concurrencyManifest, + visionManifest, ].map((m) => [m.name, m.description] as const); // Smart-scroll: keep the transcript pinned to the bottom while it streams, @@ -279,8 +287,8 @@ store.invoke(msg.surfaceId, msg.actionId, msg.payload); } - function handleSend(text: string) { - store.send(text); + function handleSend(text: string, images?: readonly ImageInput[]): void { + store.send(text, images); } function handleQueue(text: string) { @@ -342,6 +350,27 @@ : { ok: false, error: result.error }; } + // Adapt the store's global vision-settings API to the vision feature's ports. + async function loadVisionSettings(): Promise<LoadVisionSettingsResult> { + // The store seeds `visionSettings` on boot; a refresh keeps it current. + await store.refreshVisionSettings(); + const settings = store.visionSettings; + if (settings === null) { + return { ok: false, error: "Vision settings not available." }; + } + return { ok: true, settings }; + } + + async function saveVisionSettings( + patch: VisionSettingsPatch, + ): Promise<SaveVisionSettingsResult> { + const result = await store.setVisionSettings(patch); + if (result === null) return { ok: false, error: "Vision settings not available." }; + return result.ok + ? { ok: true, settings: result.settings } + : { ok: false, error: result.error }; + } + // Adapt the store's chat-limit result to the settings feature's port. On a // raise the active chat refills (prepends older history); preserve the // reader's viewport over the prepend (the manual analogue of CSS scroll @@ -572,6 +601,7 @@ onShowEarlier={handleShowEarlier} thinkingKeyBase={store.activeChat.thinkingKeyBase} providerRetry={store.activeChat.providerRetry} + apiBaseUrl={store.httpBase} /> {/key} </div> @@ -662,6 +692,7 @@ closeChat={closeRunChat} stopRun={stopHeartbeatRun} onClose={() => (heartbeatRun = null)} + apiBaseUrl={store.httpBase} /> {/key} {/if} @@ -684,7 +715,12 @@ {/key} {:else if kind === "model"} <div class="flex flex-col gap-3"> - <ModelSelector models={store.models} selected={store.activeModel} onSelect={handleSelectModel} /> + <ModelSelector + models={store.models} + selected={store.activeModel} + onSelect={handleSelectModel} + modelInfo={store.modelInfo} + /> <!-- Keyed on the workspace conversation (active tab OR draft) so the inputs re-mount per conversation — incl. switching between drafts — and can't bleed across tabs. Editable for a draft too (cwd + effort apply from turn 1). --> @@ -743,7 +779,9 @@ {/if} {/key} {:else if kind === "compaction"} - <!-- Re-mount per conversation so the percent + feedback can't bleed across tabs. --> + <!-- Message compaction is per-conversation (keyed so the percent + feedback + can't bleed across tabs). Vision (image-compaction) settings are GLOBAL, + so they stay mounted across conversation switches (no {#key}). --> {#key store.currentConversationId} <CompactionView percent={store.compactPercent} @@ -752,6 +790,13 @@ savePercent={saveCompactPercent} /> {/key} + <div class="divider my-1 text-xs text-base-content/40">Image compaction</div> + <VisionSettingsView + models={store.models} + modelInfo={store.modelInfo} + load={loadVisionSettings} + save={saveVisionSettings} + /> {:else if kind === "system-prompt"} <!-- Global system prompt template. Opens a full-page modal editor (half template / half variable palette). Not conversation-scoped (no {#key}). --> diff --git a/src/app/store.svelte.ts b/src/app/store.svelte.ts index 0506f91..ead27a6 100644 --- a/src/app/store.svelte.ts +++ b/src/app/store.svelte.ts @@ -27,6 +27,7 @@ import type { SetReasoningEffortRequest, SetSystemPromptTemplateRequest, SetTitleRequest, + SetVisionSettingsRequest, SystemPromptTemplateResponse, SystemPromptVariable, SystemPromptVariablesResponse, @@ -35,7 +36,7 @@ import type { WarmResponse, } from "@dispatch/transport-contract"; import type { SubscribeMessage, SurfaceServerMessage, SurfaceSpec } from "@dispatch/ui-contract"; -import type { ComputerEntry, ConversationStatus } from "@dispatch/wire"; +import type { ComputerEntry, ConversationStatus, ImageInput } from "@dispatch/wire"; import { untrack } from "svelte"; import { createIdbChunkStore } from "../adapters/idb"; import { createLocalStore } from "../adapters/local-storage"; @@ -78,6 +79,11 @@ import type { import { normalizeHeartbeatConfig, normalizeHeartbeatRuns } from "../features/heartbeat"; import type { Tab, TabsState } from "../features/tabs"; import { createTabsStore, deriveTitle, type TabsStore } from "../features/tabs"; +import { + normalizeVisionSettings, + type VisionSettings, + type VisionSettingsPatch, +} from "../features/vision"; import { resolveHttpUrl } from "./resolve-http-url"; import { resolveWsUrl } from "./resolve-ws-url"; import { randomId } from "./uuid"; @@ -134,6 +140,11 @@ export type CompactPercentResult = | { readonly ok: true; readonly percent: number } | { readonly ok: false; readonly error: string }; +/** Outcome of `PUT /settings/vision` (global vision-settings save). */ +export type VisionSettingsResult = + | { readonly ok: true; readonly settings: VisionSettings } + | { readonly ok: false; readonly error: string }; + /** Outcome of `GET /system-prompt` (global template load). */ export type SystemPromptLoadResult = | { readonly ok: true; readonly template: string } @@ -157,6 +168,12 @@ export interface AppStore { readonly activeConversationId: string | null; /** The workspace currently in view (URL slug); tabs are filtered to it. */ readonly activeWorkspaceId: string; + /** + * The resolved HTTP API base URL (e.g. `http://localhost:24203`). Used to + * resolve relative image URLs served by the backend (`/images/…`) into + * absolute URLs for `<img src>`. + */ + readonly httpBase: string; readonly activeChat: ChatStore; readonly models: readonly string[]; /** Per-model metadata (contextWindow, etc.) from `GET /models`. */ @@ -171,7 +188,14 @@ export interface AppStore { readonly storage: Storage | undefined; /** The current spec for one surface by id (discovery-by-id), or null if absent. */ surface(surfaceId: string): SurfaceSpec | null; - send(text: string): void; + /** + * Send a user message (start a turn). Forwards any staged `images` + * (`ImageInput[]` — base64 data URLs / https URLs) on the `chat.send` op; + * the server passes them to a vision-capable model natively or transcribes + * them via vision handoff for a non-vision model. Omitted on the wire when + * none are staged. On a draft, promotes to a tab first. + */ + send(text: string, images?: readonly ImageInput[]): void; /** * Enqueue a steering message onto the focused conversation's queue * (`chat.queue` WS op). While a turn is generating, the message is delivered @@ -273,6 +297,23 @@ export interface AppStore { */ setCompactPercent(percent: number): Promise<CompactPercentResult | null>; /** + * The GLOBAL vision settings (`GET /settings/vision`): `imageLimit` (max + * native images per turn before compaction; 0 = disabled) + `compactionModel` + * (which vision model transcribes old images; null = auto). Shared across all + * conversations. Seeded on boot; `null` = not yet fetched. + */ + readonly visionSettings: VisionSettings | null; + /** + * Refetch the global vision settings (`GET /settings/vision`). Called by the + * vision-settings view on mount; also seeded on boot. + */ + refreshVisionSettings(): Promise<void>; + /** + * Save a PARTIAL vision-settings update (`PUT /settings/vision`). Either + * field may be omitted. Returns the merged settings on success. + */ + setVisionSettings(patch: VisionSettingsPatch): Promise<VisionSettingsResult | null>; + /** * Fetch the workspace conversation's language-server status (`GET /conversations/:id/lsp`). * The backend lazily spawns servers, so this may take a moment on the first call for a cwd. */ @@ -662,6 +703,22 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { // backend on focus change; null = not yet fetched. 0 = disabled. let compactPercent = $state<number | null>(null); + // The GLOBAL vision settings (shared across all conversations). Seeded on + // boot; null = not yet fetched. + let visionSettings = $state<VisionSettings | null>(null); + + /** Refetch the global vision settings (`GET /settings/vision`). */ + async function refreshVisionSettings(): Promise<void> { + try { + const res = await fetchImpl(`${httpBase}/settings/vision`); + if (!res.ok) return; + const data = normalizeVisionSettings(await res.json()); + visionSettings = data; + } catch (err) { + reportError("Failed to load vision settings", err); + } + } + /** Refetch the workspace conversation's compact percent (works for a draft too). */ async function refreshCompactPercent(): Promise<void> { const id = workspaceConversationId(); @@ -1106,6 +1163,7 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { void refreshModel(); void refreshReasoningEffort(); void refreshCompactPercent(); + void refreshVisionSettings(); // Fetch the authoritative open-conversation list from the backend (cross- // device tab sync). Merges with the localStorage-restored tabs: opens new @@ -1122,6 +1180,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { get activeWorkspaceId(): string { return activeWorkspaceId; }, + get httpBase(): string { + return httpBase; + }, setActiveWorkspace(workspaceId: string): void { activeWorkspaceId = workspaceId; // Reset to a fresh draft scoped to the new workspace so a new chat is @@ -1182,6 +1243,12 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { get compactPercent(): number | null { return compactPercent; }, + get visionSettings(): VisionSettings | null { + return visionSettings; + }, + async refreshVisionSettings(): Promise<void> { + await refreshVisionSettings(); + }, get chatLimit(): number { return chatLimit; }, @@ -1196,7 +1263,7 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { return getSurfaceSpec(protocol, surfaceId); }, - send(text: string): void { + send(text: string, images?: readonly ImageInput[]): void { if (tabsStore.activeConversationId === null) { // Draft: promote to tab on first send const conversationId = draftConversationId; @@ -1224,9 +1291,9 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { void refreshReasoningEffort(); void refreshCompactPercent(); // Now send on the promoted store - chatStores.get(conversationId)?.send(text); + chatStores.get(conversationId)?.send(text, images); } else { - activeChat.send(text); + activeChat.send(text, images); } }, @@ -1540,6 +1607,32 @@ export function createAppStore(opts?: CreateAppStoreOptions): AppStore { } }, + async setVisionSettings(patch: VisionSettingsPatch): Promise<VisionSettingsResult | null> { + const body: SetVisionSettingsRequest = patch; + try { + const res = await fetchImpl(`${httpBase}/settings/vision`, { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) { + const errBody = (await res.json().catch(() => null)) as { error?: string } | null; + return { + ok: false, + error: errBody?.error ?? `Set vision settings failed (HTTP ${res.status})`, + }; + } + const data = normalizeVisionSettings(await res.json()); + visionSettings = data; + return { ok: true, settings: data }; + } catch (err) { + return { + ok: false, + error: err instanceof Error ? err.message : "Set vision settings request failed", + }; + } + }, + async setChatLimit(limit: number): Promise<ChatLimitResult> { const next = normalizeChatLimit(limit); chatLimitStore.save(next); diff --git a/src/app/store.test.ts b/src/app/store.test.ts index b47a378..2cf473c 100644 --- a/src/app/store.test.ts +++ b/src/app/store.test.ts @@ -423,6 +423,34 @@ describe("createAppStore", () => { store.dispose(); }); + it("sending from draft forwards staged images on chat.send", () => { + const ws = fakeSocket(); + const store = createAppStore({ + socketFactory: () => ws, + fetchImpl: fakeFetchImpl(), + localStorage: createFakeStorage(), + }); + ws.resolveOpen(); + ws.sent.length = 0; + + const images = [ + { url: "data:image/png;base64,AAAA", mimeType: "image/png" }, + { url: "https://example.com/x.jpg" }, + ]; + store.send("describe these", images); + + const msgs = parseSent(ws); + const chatSend = msgs.find((m) => (m as { type: string }).type === "chat.send") as + | { type: string; message: string; images?: { url: string }[] } + | undefined; + expect(chatSend).toBeTruthy(); + expect(chatSend?.images).toEqual(images); + // The optimistic echo includes the image chunks. + expect(store.activeChat.chunks.some((c) => c.chunk.type === "image")).toBe(true); + + store.dispose(); + }); + it("an incoming chat.delta renders in the transcript", () => { const ws = fakeSocket(); const store = createAppStore({ @@ -1703,3 +1731,127 @@ describe("createAppStore", () => { store.dispose(); }); }); + +describe("createAppStore — vision settings (global)", () => { + function visionFetch(initial: { imageLimit: number; compactionModel: string | null }): { + fetchImpl: typeof fetch; + puts: { imageLimit?: number; compactionModel?: string | null }[]; + } { + let current = initial; + const puts: { imageLimit?: number; compactionModel?: string | null }[] = []; + return { + puts, + fetchImpl: async (input: string | URL | Request, init?: RequestInit): Promise<Response> => { + const url = + typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.endsWith("/models")) { + return new Response( + JSON.stringify({ + models: ["kimi/k2", "umans/glm-5.2"], + modelInfo: { "kimi/k2": { vision: true } }, + }), + { status: 200 }, + ); + } + if (url.endsWith("/settings/vision")) { + if (init?.method === "PUT") { + const text = typeof init.body === "string" ? init.body : ""; + const body = text ? (JSON.parse(text) as object) : {}; + puts.push(body as { imageLimit?: number; compactionModel?: string | null }); + current = { ...current, ...(body as object) } as { + imageLimit: number; + compactionModel: string | null; + }; + } + return new Response(JSON.stringify(current), { status: 200 }); + } + // Default: empty history + no cwd for the other endpoints. + return new Response(JSON.stringify({ chunks: [], latestSeq: 0 }), { status: 200 }); + }, + }; + } + + it("loads vision settings on boot (GET /settings/vision)", async () => { + const { fetchImpl } = visionFetch({ imageLimit: 7, compactionModel: "kimi/k2" }); + const store = createAppStore({ + socketFactory: () => fakeSocket(), + fetchImpl, + localStorage: createFakeStorage(), + }); + fakeSocket().resolveOpen(); // not strictly needed for HTTP + + await vi.waitFor(() => { + expect(store.visionSettings).toEqual({ imageLimit: 7, compactionModel: "kimi/k2" }); + }); + store.dispose(); + }); + + it("setVisionSettings PUTs a partial update and reflects the merged settings", async () => { + const ctx = visionFetch({ imageLimit: 10, compactionModel: null }); + const store = createAppStore({ + socketFactory: () => fakeSocket(), + fetchImpl: ctx.fetchImpl, + localStorage: createFakeStorage(), + }); + + await vi.waitFor(() => { + expect(store.visionSettings?.imageLimit).toBe(10); + }); + + const result = await store.setVisionSettings({ imageLimit: 3 }); + expect(result?.ok).toBe(true); + if (result?.ok) { + expect(result.settings.imageLimit).toBe(3); + expect(result.settings.compactionModel).toBeNull(); + } + expect(ctx.puts).toEqual([{ imageLimit: 3 }]); + expect(store.visionSettings?.imageLimit).toBe(3); + + // A second save updates compactionModel only. + const result2 = await store.setVisionSettings({ compactionModel: "kimi/k2" }); + expect(result2?.ok).toBe(true); + expect(ctx.puts).toEqual([{ imageLimit: 3 }, { compactionModel: "kimi/k2" }]); + expect(store.visionSettings?.compactionModel).toBe("kimi/k2"); + store.dispose(); + }); + + it("refreshVisionSettings refetches (load adapter)", async () => { + const { fetchImpl } = visionFetch({ imageLimit: 5, compactionModel: null }); + const store = createAppStore({ + socketFactory: () => fakeSocket(), + fetchImpl, + localStorage: createFakeStorage(), + }); + + await store.refreshVisionSettings(); + expect(store.visionSettings).toEqual({ imageLimit: 5, compactionModel: null }); + store.dispose(); + }); + + it("surfaces a PUT error", async () => { + const fetchImpl: typeof fetch = async (input, init) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.endsWith("/settings/vision")) { + if (init?.method === "PUT") { + return new Response(JSON.stringify({ error: "invalid imageLimit" }), { status: 400 }); + } + return new Response(JSON.stringify({ imageLimit: 10, compactionModel: null }), { + status: 200, + }); + } + return new Response(JSON.stringify({ models: [] }), { status: 200 }); + }; + const store = createAppStore({ + socketFactory: () => fakeSocket(), + fetchImpl, + localStorage: createFakeStorage(), + }); + + const result = await store.setVisionSettings({ imageLimit: -1 }); + expect(result?.ok).toBe(false); + if (result !== null && !result.ok) { + expect(result.error).toContain("invalid imageLimit"); + } + store.dispose(); + }); +}); |