diff options
| author | Adam Malczewski <[email protected]> | 2026-06-03 13:02:15 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-03 13:02:15 +0900 |
| commit | e87e6b39285c8001045d1ebdac873b182c0f7868 (patch) | |
| tree | 27003852f7b182fd65c6ad762784aa5fcf839ebc | |
| parent | ae672fd4f5542a2c217cf97657bf81eeebdaabbd (diff) | |
| download | dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.tar.gz dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.zip | |
feat: prompt cache warming for idle tabs
Keep a tab's provider prompt-cache warm while idle by periodically replaying
the exact cached conversation prefix plus a single trivial throwaway turn,
resetting the provider's ~5-min cache TTL so the user's next real message hits
a warm cache.
Backend:
- Agent.warmCache(history): extracts buildLlmContext() shared with run(), then
re-sends the identical system+tools+history prefix (same Anthropic
cache_control breakpoints) plus a 'reply with just a .' probe turn via
toolChoice:none. Returns the request usage; mutates no history, emits/persists
nothing.
- AgentManager.warmCacheForTab(): resolves the same agent the next real turn
would use, replays the FULL genuine history, refuses while a turn is running.
- POST /chat/warm: returns ONLY the warming request's usage (never persisted,
never folded into the real usage aggregate).
Frontend:
- cache-warming.svelte.ts store: per-tab 4-min repeating idle timer with
countdown, warming-specific last-request cache %, and error capture. Arms on
turn end, pauses during a turn, disables+resets on a real user message.
- cache-warm-storage.ts: per-tab localStorage persistence of the toggle.
- Lifecycle hooks wired into tabs.svelte.ts (status/statuses/sendMessage/
hydrate/create/open/close).
- ModelSelector: bottom-of-panel checkbox + debug strip (last-% / countdown /
error), shown only when enabled. Warming cache data never touches the real
Cache Rate view.
Tests: core warmCache (5), api warm route (3) + warmCacheForTab (3), frontend
store (12) + storage (10). check / test (779) / frontend build / typecheck all
green.
| -rw-r--r-- | packages/api/src/agent-manager.ts | 60 | ||||
| -rw-r--r-- | packages/api/src/app.ts | 37 | ||||
| -rw-r--r-- | packages/api/tests/agent-manager.test.ts | 64 | ||||
| -rw-r--r-- | packages/api/tests/routes.test.ts | 53 | ||||
| -rw-r--r-- | packages/core/src/agent/agent.ts | bin | 60515 -> 65763 bytes | |||
| -rw-r--r-- | packages/core/tests/agent/agent.test.ts | 107 | ||||
| -rw-r--r-- | packages/frontend/src/App.svelte | 1 | ||||
| -rw-r--r-- | packages/frontend/src/lib/cache-warm-storage.ts | 77 | ||||
| -rw-r--r-- | packages/frontend/src/lib/cache-warming.svelte.ts | 311 | ||||
| -rw-r--r-- | packages/frontend/src/lib/components/ModelSelector.svelte | 89 | ||||
| -rw-r--r-- | packages/frontend/src/lib/components/SidebarPanel.svelte | 3 | ||||
| -rw-r--r-- | packages/frontend/src/lib/tabs.svelte.ts | 49 | ||||
| -rw-r--r-- | packages/frontend/tests/cache-warm-storage.test.ts | 111 | ||||
| -rw-r--r-- | packages/frontend/tests/cache-warming.test.ts | 219 |
14 files changed, 1181 insertions, 0 deletions
diff --git a/packages/api/src/agent-manager.ts b/packages/api/src/agent-manager.ts index c1b46b9..109dd33 100644 --- a/packages/api/src/agent-manager.ts +++ b/packages/api/src/agent-manager.ts @@ -8,6 +8,7 @@ import { appendEventToChunks, BackgroundShellStore, BackgroundTranscriptStore, + type ChatMessage, type Chunk, type ClaudeAccount, clearSpillForTab, @@ -1019,6 +1020,65 @@ export class AgentManager { } /** + * Prompt-cache WARMING for an idle tab (see `Agent.warmCache`). + * + * Reconstructs the tab's genuine conversation from the persisted chunk log, + * resolves the SAME agent (model/key/tools/system prompt) the next real turn + * would use, and replays the exact cached prefix plus one trivial throwaway + * turn so the provider's ~5-min prompt-cache TTL is refreshed. The warming + * request and its response are NOT persisted, NOT emitted, and NOT folded + * into the real usage aggregate — its `usage` is returned to the caller so a + * warming-only "last request" cache rate can be shown without polluting the + * real Cache Rate metric. + * + * Refuses to fire while the tab is generating (`running`): the prefix would + * be mid-mutation and the request would contend with the live turn. Callers + * gate on idle anyway; this is defence in depth. + * + * Returns `{ ok: true, usage }` on success or `{ ok: false, error }` so the + * route can surface a debug-strip error string. Never throws. + */ + async warmCacheForTab( + tabId: string, + opts: { keyId?: string; modelId?: string; agentModels?: AgentModelEntry[] } = {}, + ): Promise<{ ok: true; usage: UsageData } | { ok: false; error: string }> { + if (this.getTabStatus(tabId) === "running") { + return { ok: false, error: "tab is generating" }; + } + try { + const tabAgent = this._getOrCreateTabAgent(tabId); + if (opts.agentModels) tabAgent.agentModels = opts.agentModels; + + // Resolve the agent the next REAL turn would use. The fallback chain's + // first entry mirrors `processMessage`'s primary attempt; we only warm + // the primary (warming a fallback model would write a DIFFERENT prefix). + const fallbackSequence = this.buildFallbackSequence(tabAgent, opts.keyId, opts.modelId); + const primary = fallbackSequence[0]; + const agent = await this.getOrCreateAgentForTab( + tabId, + primary?.key_id || opts.keyId, + primary?.model_id || opts.modelId, + ); + + // Rebuild the genuine history exactly as `getOrCreateAgentForTab`'s + // pre-population does, but keep the FULL history (no trailing-user + // trim): warming replays the complete cached prefix as-is. + let history: ChatMessage[] = []; + try { + history = getMessagesForTab(tabId).map((r) => ({ role: r.role, chunks: r.chunks })); + } catch { + // DB read failed — warm with whatever in-memory history the agent has. + history = [...agent.messages]; + } + + const usage = await agent.warmCache(history); + return { ok: true, usage }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + /** * Snapshot of every tab the manager is currently tracking. Sent on WS * connect and via GET /status so a freshly-loaded frontend can * reconstruct any in-flight assistant turn without missing the chunks diff --git a/packages/api/src/app.ts b/packages/api/src/app.ts index 2f4e538..a957da7 100644 --- a/packages/api/src/app.ts +++ b/packages/api/src/app.ts @@ -224,6 +224,43 @@ app.post("/chat/stop", async (c) => { return c.json({ success: true }); }); +// Prompt-cache WARMING (see AgentManager.warmCacheForTab / Agent.warmCache). +// +// Replays the tab's exact cached prefix + one trivial throwaway turn so the +// provider's ~5-min prompt-cache TTL is refreshed while the tab sits idle. +// The frontend's cache-warming timer drives this every ~4 minutes. The +// warming request is NEVER persisted, NEVER emitted, and NEVER folded into the +// real usage aggregate — we return ONLY its `usage` so the UI can show a +// warming-specific "last request" cache rate without polluting the real +// Cache Rate metric. Returns 409 when the tab is mid-turn (caller also gates). +app.post("/chat/warm", async (c) => { + const body = await c.req.json<{ + tabId?: unknown; + keyId?: unknown; + modelId?: unknown; + agentModels?: unknown; + }>(); + const { tabId } = body; + if (typeof tabId !== "string" || tabId.trim() === "") { + return c.json({ error: "tabId must be a non-empty string" }, 400); + } + const keyId = typeof body.keyId === "string" ? body.keyId : undefined; + const modelId = typeof body.modelId === "string" ? body.modelId : undefined; + const agentModels = sanitizeAgentModels(body.agentModels); + + const result = await agentManager.warmCacheForTab(tabId, { + ...(keyId ? { keyId } : {}), + ...(modelId ? { modelId } : {}), + ...(agentModels ? { agentModels } : {}), + }); + if (!result.ok) { + // "tab is generating" is an expected race (not a server fault) → 409. + const status = result.error === "tab is generating" ? 409 : 500; + return c.json({ error: result.error }, status); + } + return c.json({ usage: result.usage }); +}); + app.route("/skills", skillsRoutes); app.route("/models", modelsRoutes); app.route("/tabs", tabsRoutes); diff --git a/packages/api/tests/agent-manager.test.ts b/packages/api/tests/agent-manager.test.ts index 788106e..707039a 100644 --- a/packages/api/tests/agent-manager.test.ts +++ b/packages/api/tests/agent-manager.test.ts @@ -91,6 +91,13 @@ function resetCapturedRunOptions(): void { capturedRunOptions.length = 0; } +// Capture every warmCache(history) call so tests can assert the warming replay +// receives the genuine (FULL) history and returns its usage unmodified. +const capturedWarmHistories: unknown[][] = []; +function resetCapturedWarmHistories(): void { + capturedWarmHistories.length = 0; +} + // Configurable settings store so tests can toggle tool permissions // (perm_send_to_tab / perm_read_tab / ...) and assert which tools the // constructed Agent receives. Defaults to empty (getSetting → null). @@ -185,6 +192,15 @@ vi.mock("@dispatch/core", () => ({ } for await (const ev of defaultRun(message)) yield ev; } + async warmCache(history: unknown[]) { + capturedWarmHistories.push([...history]); + return { + inputTokens: 1200, + outputTokens: 1, + cacheReadTokens: 1100, + cacheWriteTokens: 0, + }; + } }, PermissionService: class MockPermissionService { ask(_request: unknown, _rulesets: unknown[]) { @@ -512,6 +528,7 @@ describe("AgentManager", () => { appendEventToChunksSpy.mockClear(); resetAppendChunksCalls(); resetFakeUsageStats(); + resetCapturedWarmHistories(); }); it("initial status is idle", () => { @@ -1858,4 +1875,51 @@ describe("AgentManager", () => { }); }); }); + + describe("warmCacheForTab (prompt-cache warming)", () => { + it("returns the warm request usage and forwards the FULL genuine history", async () => { + const manager = new AgentManager(); + setFakeMessages("tab-warm", [ + makeRow("tab-warm", 1, "user", [{ type: "text", text: "hello" }]), + makeRow("tab-warm", 2, "assistant", [{ type: "text", text: "hi" }]), + ]); + + const result = await manager.warmCacheForTab("tab-warm"); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.usage).toEqual({ + inputTokens: 1200, + outputTokens: 1, + cacheReadTokens: 1100, + cacheWriteTokens: 0, + }); + } + // The genuine history is forwarded UNTRIMMED (both turns), so the + // replayed prefix matches the next real turn exactly. + expect(capturedWarmHistories).toHaveLength(1); + expect(capturedWarmHistories[0]).toHaveLength(2); + }); + + it("does NOT persist anything (no appendChunks for the warm request)", async () => { + const manager = new AgentManager(); + setFakeMessages("tab-warm-2", [ + makeRow("tab-warm-2", 1, "user", [{ type: "text", text: "hello" }]), + ]); + await manager.warmCacheForTab("tab-warm-2"); + // Warming must never write chunk rows (history / usage / anything). + expect(appendChunksCalls).toHaveLength(0); + }); + + it("refuses to warm while the tab is generating", async () => { + const manager = new AgentManager(); + // Start a turn (status flips to running) but don't await it. + const running = manager.processMessage("tab-warm-busy", "go"); + // Let the mock run() yield its first running status. + await new Promise<void>((r) => setTimeout(r, 1)); + const result = await manager.warmCacheForTab("tab-warm-busy"); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toBe("tab is generating"); + await running; + }); + }); }); diff --git a/packages/api/tests/routes.test.ts b/packages/api/tests/routes.test.ts index 7cfd8a7..d2c8229 100644 --- a/packages/api/tests/routes.test.ts +++ b/packages/api/tests/routes.test.ts @@ -24,6 +24,15 @@ vi.mock("@dispatch/core", () => ({ Agent: class MockAgent { status = "idle"; messages: unknown[] = []; + async warmCache(_history: unknown[]) { + // Simulate a warm replay that read most of the prompt from cache. + return { + inputTokens: 1000, + outputTokens: 1, + cacheReadTokens: 900, + cacheWriteTokens: 0, + }; + } async *run(_message: string) { yield { type: "status", status: "running" } as const; // Simulate some processing time so status stays "running" @@ -665,6 +674,50 @@ describe("POST /chat/stop", () => { expect(res.status).toBe(400); }); }); + +describe("POST /chat/warm", () => { + it("returns ONLY the warming request usage (never persisted/emitted)", async () => { + const res = await app.request("/chat/warm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ tabId: "tab-warm-1" }), + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { usage?: Record<string, number> }; + expect(body.usage).toEqual({ + inputTokens: 1000, + outputTokens: 1, + cacheReadTokens: 900, + cacheWriteTokens: 0, + }); + }); + + it("returns 400 when tabId is missing", async () => { + const res = await app.request("/chat/warm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + expect(res.status).toBe(400); + }); + + it("returns 409 while the tab is generating", async () => { + // Kick off a real (mock) turn so the tab is "running", then immediately + // attempt to warm it — warming must refuse mid-turn. + await app.request("/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ tabId: "tab-warm-busy", message: "hi" }), + }); + const res = await app.request("/chat/warm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ tabId: "tab-warm-busy" }), + }); + expect(res.status).toBe(409); + }); +}); + describe("Wake schedule routes", () => { async function getSchedule() { const res = await app.request("/models/wake-schedule"); diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts Binary files differindex 08b317a..d0a3bb9 100644 --- a/packages/core/src/agent/agent.ts +++ b/packages/core/src/agent/agent.ts diff --git a/packages/core/tests/agent/agent.test.ts b/packages/core/tests/agent/agent.test.ts index f4b33cc..86a7a5b 100644 --- a/packages/core/tests/agent/agent.test.ts +++ b/packages/core/tests/agent/agent.test.ts @@ -1642,4 +1642,111 @@ describe("anthropicThinkingProviderOptions — adaptive-thinking model detection expect(userMsg?.content).toBe("plain text"); }); }); + + describe("warmCache (prompt-cache warming replay)", () => { + function makeWarmStream(usage: { + inputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + }) { + return makeMockStreamResult([ + { type: "text-delta", id: "t0", text: "." }, + { + type: "finish-step", + finishReason: "stop", + rawFinishReason: "stop", + usage: { + inputTokens: usage.inputTokens, + outputTokens: 1, + inputTokenDetails: { + noCacheTokens: usage.inputTokens - usage.cacheReadTokens - usage.cacheWriteTokens, + cacheReadTokens: usage.cacheReadTokens, + cacheWriteTokens: usage.cacheWriteTokens, + }, + }, + }, + finishStop, + ]); + } + + const history = [ + { role: "user" as const, chunks: [{ type: "text" as const, text: "hello" }] }, + { role: "assistant" as const, chunks: [{ type: "text" as const, text: "hi there" }] }, + ]; + + it("returns the request usage (cache read/write split) without throwing", async () => { + vi.mocked(streamText).mockReturnValue( + makeWarmStream({ inputTokens: 1000, cacheReadTokens: 950, cacheWriteTokens: 0 }), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + const usage = await agent.warmCache(history); + expect(usage).toEqual({ + inputTokens: 1000, + outputTokens: 1, + cacheReadTokens: 950, + cacheWriteTokens: 0, + }); + }); + + it("appends a single trivial throwaway user turn at the END of the history", async () => { + vi.mocked(streamText).mockReturnValue( + makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + await agent.warmCache(history); + + const callArgs = vi.mocked(streamText).mock.calls.at(-1)?.[0]; + const messages = callArgs?.messages as Array<{ role: string; content: unknown }>; + // system + 2 history messages + 1 throwaway user turn. + expect(messages[0]?.role).toBe("system"); + const last = messages.at(-1); + expect(last?.role).toBe("user"); + // The throwaway turn's text must be the trivial probe. + const lastText = JSON.stringify(last?.content); + expect(lastText).toContain("reply with just a ."); + // Exactly one extra user turn beyond the genuine history's single user msg. + const userMsgs = messages.filter((m) => m.role === "user"); + expect(userMsgs).toHaveLength(2); + }); + + it("sends Anthropic cache_control breakpoints + toolChoice none", async () => { + vi.mocked(streamText).mockReturnValue( + makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + await agent.warmCache(history); + + const callArgs = vi.mocked(streamText).mock.calls.at(-1)?.[0]; + expect(callArgs?.toolChoice).toBe("none"); + const messages = callArgs?.messages as Array<{ + role: string; + providerOptions?: { anthropic?: { cacheControl?: unknown } }; + }>; + const hasBreakpoint = messages.some( + (m) => m.providerOptions?.anthropic?.cacheControl !== undefined, + ); + expect(hasBreakpoint).toBe(true); + }); + + it("does NOT mutate the agent's own message history", async () => { + vi.mocked(streamText).mockReturnValue( + makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + expect(agent.messages).toHaveLength(0); + await agent.warmCache(history); + // warmCache takes history as an argument and never touches `this.messages`. + expect(agent.messages).toHaveLength(0); + // And it must not have flipped the agent into a running state. + expect(agent.status).toBe("idle"); + }); + + it("throws a formatted error when the stream errors", async () => { + vi.mocked(streamText).mockReturnValue( + makeMockStreamResult([{ type: "error", error: new Error("boom") }]), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + await expect(agent.warmCache(history)).rejects.toThrow(/boom/); + }); + }); }); diff --git a/packages/frontend/src/App.svelte b/packages/frontend/src/App.svelte index ae0718e..3a2d27c 100644 --- a/packages/frontend/src/App.svelte +++ b/packages/frontend/src/App.svelte @@ -250,6 +250,7 @@ onMount(() => { {contextLimit} permissionLog={tabStore.permissionLog} apiBase={config.apiBase} + activeTabId={tabStore.activeTabId} activeKeyId={tabStore.activeTab?.keyId ?? null} activeModelId={tabStore.activeTab?.modelId ?? null} reasoningEffort={tabStore.activeTab?.reasoningEffort ?? DEFAULT_REASONING_EFFORT} diff --git a/packages/frontend/src/lib/cache-warm-storage.ts b/packages/frontend/src/lib/cache-warm-storage.ts new file mode 100644 index 0000000..66a8c31 --- /dev/null +++ b/packages/frontend/src/lib/cache-warm-storage.ts @@ -0,0 +1,77 @@ +/** + * LocalStorage persistence for the per-tab "prompt cache warming" toggle. + * + * Cache warming keeps a tab's provider prompt-cache warm while the tab is idle + * by periodically replaying its exact cached conversation plus a trivial + * throwaway turn (see `cache-warming.svelte.ts`). Whether warming is enabled is + * a per-tab preference that must survive a browser reload. + * + * Why localStorage and not the backend `settings` table: + * - It's a per-device UI preference, not domain state — the same precedent as + * `dispatch-sidebar-panels`, `dispatch-theme`, and `dispatch-api-url`. + * - No backend round-trip on every toggle. + * - Warming itself is a frontend-driven timer; keeping its on/off flag on the + * frontend keeps the whole feature self-contained. + * + * Shape: a single JSON object mapping `tabId -> boolean` under one key, so a + * closed tab's stale entry is cheap and easy to prune. + */ + +const LS_KEY = "dispatch-cache-warming"; + +type WarmMap = Record<string, boolean>; + +/** Read the whole tab→enabled map. Never throws; returns {} on any failure. */ +function readMap(): WarmMap { + try { + const raw = localStorage.getItem(LS_KEY); + if (!raw) return {}; + const parsed: unknown = JSON.parse(raw); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {}; + const out: WarmMap = {}; + for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) { + if (typeof v === "boolean") out[k] = v; + } + return out; + } catch { + // localStorage unavailable (private mode / restricted context) or corrupt + // write from a prior session. Degrade to "nothing persisted". + return {}; + } +} + +/** Persist the whole map. Best-effort — swallows quota / access errors. */ +function writeMap(map: WarmMap): void { + try { + localStorage.setItem(LS_KEY, JSON.stringify(map)); + } catch { + // Best-effort: the in-memory store remains the source of truth for the + // current session even if persistence fails. + } +} + +/** Whether cache warming is enabled for `tabId` (default: false). */ +export function loadCacheWarmEnabled(tabId: string): boolean { + return readMap()[tabId] === true; +} + +/** + * Persist the warming toggle for one tab. Writing `false` removes the entry + * (default is off, so absence is the natural "disabled" representation and + * keeps stale tabs from accumulating). + */ +export function saveCacheWarmEnabled(tabId: string, enabled: boolean): void { + const map = readMap(); + if (enabled) map[tabId] = true; + else delete map[tabId]; + writeMap(map); +} + +/** Drop a tab's persisted toggle entirely (called when the tab is closed). */ +export function clearCacheWarmEnabled(tabId: string): void { + const map = readMap(); + if (tabId in map) { + delete map[tabId]; + writeMap(map); + } +} diff --git a/packages/frontend/src/lib/cache-warming.svelte.ts b/packages/frontend/src/lib/cache-warming.svelte.ts new file mode 100644 index 0000000..cda3fd1 --- /dev/null +++ b/packages/frontend/src/lib/cache-warming.svelte.ts @@ -0,0 +1,311 @@ +/** + * Prompt-cache WARMING — frontend timer/orchestration store. + * + * Keeps a tab's provider prompt-cache warm while the tab is IDLE by firing a + * cheap "warm" request (`POST /chat/warm`) on a repeating ~4-minute cadence. + * The backend replays the tab's EXACT cached prefix plus one trivial throwaway + * turn (see `Agent.warmCache`), which registers a cache READ and refreshes the + * provider's ~5-min prompt-cache TTL so the user's next real message lands on a + * warm cache. + * + * Lifecycle (driven by the tab store via the `onTurn*` / `onUserMessage` hooks): + * - A turn ENDS (tab goes idle) → arm: schedule a fire in 4 minutes. + * - The timer fires → warm, then re-arm 4 minutes out + * (repeats; resets the countdown each + * cycle). + * - A turn is ONGOING (generation active) → never fires; the pending timer is + * cancelled. + * - The user sends a real message → disable+reset the timer immediately; + * the turn it starts re-arms warming + * once it ends. + * + * CRITICAL: the warming request is debug-only. Its cache data is surfaced ONLY + * as a warming-specific "Last request" percentage here — it is NEVER folded + * into the real Cache Rate metric, never persisted, never counted toward + * context. The backend route returns just the request's `usage`; nothing else. + */ + +import type { AgentModelEntry } from "@dispatch/core/src/types/index.js"; +import { + clearCacheWarmEnabled, + loadCacheWarmEnabled, + saveCacheWarmEnabled, +} from "./cache-warm-storage.js"; +import { config } from "./config.js"; + +/** Re-warm cadence. Comfortably under Claude's ~5-min prompt-cache expiry. */ +export const WARM_INTERVAL_MS = 4 * 60 * 1000; + +/** Per-tab request parameters the warm POST needs (resolved from the tab). */ +export interface WarmRequestParams { + keyId: string | null; + modelId: string | null; + agentModels: AgentModelEntry[] | null; +} + +/** Reactive, per-tab warming UI state (read by the Chat Settings debug strip). */ +export interface WarmState { + /** User toggle (persisted per-tab in localStorage). */ + enabled: boolean; + /** Epoch ms of the next scheduled fire, or null when not armed. */ + nextFireAt: number | null; + /** + * Cache-read % of the most recent warming request (0–100), or null if it + * has never fired this session. Drives the "-%" → number display. + */ + lastPct: number | null; + /** Last warming error (provider/network), surfaced in the debug strip. */ + error: string | null; + /** True while a warm request is in flight. */ + firing: boolean; +} + +function defaultState(enabled: boolean): WarmState { + return { enabled, nextFireAt: null, lastPct: null, error: null, firing: false }; +} + +function computeCachePct(inputTokens: number, cacheReadTokens: number): number { + if (inputTokens <= 0) return 0; + return Math.round(Math.max(0, Math.min(1, cacheReadTokens / inputTokens)) * 100); +} + +export function createCacheWarmingStore() { + // Reactive per-tab state. Nested mutation is reactive via Svelte 5 proxies; + // new keys are assigned wholesale (also reactive). + const states = $state<Record<string, WarmState>>({}); + // Ticking clock so the countdown display refreshes once per second. Only + // ticked while at least one tab is armed (see (re)startTicker). + let now = $state(Date.now()); + + // Non-reactive bookkeeping (timers, in-flight tokens, running set, resolver). + const fireTimers = new Map<string, ReturnType<typeof setTimeout>>(); + const fireTokens = new Map<string, number>(); + const runningTabs = new Set<string>(); + let ticker: ReturnType<typeof setInterval> | null = null; + let resolveParams: ((tabId: string) => WarmRequestParams | null) | null = null; + + function ensure(tabId: string): WarmState { + let s = states[tabId]; + if (!s) { + s = defaultState(loadCacheWarmEnabled(tabId)); + states[tabId] = s; + } + return s; + } + + function anyArmed(): boolean { + for (const s of Object.values(states)) { + if (s.nextFireAt !== null) return true; + } + return false; + } + + function startTickerIfNeeded(): void { + if (ticker !== null) return; + if (typeof setInterval !== "function") return; + ticker = setInterval(() => { + now = Date.now(); + // Self-stop once nothing is armed, so we don't tick forever. + if (!anyArmed()) stopTicker(); + }, 1000); + } + + function stopTicker(): void { + if (ticker !== null) { + clearInterval(ticker); + ticker = null; + } + } + + function clearFireTimer(tabId: string): void { + const t = fireTimers.get(tabId); + if (t !== undefined) { + clearTimeout(t); + fireTimers.delete(tabId); + } + } + + /** Cancel any pending fire / in-flight request and clear the countdown. */ + function cancel(tabId: string): void { + clearFireTimer(tabId); + // Invalidate any in-flight warm so its late result is ignored. + fireTokens.set(tabId, (fireTokens.get(tabId) ?? 0) + 1); + const s = states[tabId]; + if (s) s.nextFireAt = null; + if (!anyArmed()) stopTicker(); + } + + /** Schedule the next fire 4 minutes out — only when enabled AND idle. */ + function arm(tabId: string): void { + const s = ensure(tabId); + if (!s.enabled) return; + if (runningTabs.has(tabId)) return; + clearFireTimer(tabId); + s.nextFireAt = Date.now() + WARM_INTERVAL_MS; + startTickerIfNeeded(); + if (typeof setTimeout === "function") { + fireTimers.set( + tabId, + setTimeout(() => { + fireTimers.delete(tabId); + void fire(tabId); + }, WARM_INTERVAL_MS), + ); + } + } + + /** Perform one warming request, then (if still eligible) re-arm. */ + async function fire(tabId: string): Promise<void> { + const s = ensure(tabId); + if (!s.enabled || runningTabs.has(tabId) || s.firing) { + return; + } + const token = (fireTokens.get(tabId) ?? 0) + 1; + fireTokens.set(tabId, token); + const params = resolveParams?.(tabId) ?? null; + + s.firing = true; + s.error = null; + // Clear the countdown while the request is in flight. + s.nextFireAt = null; + try { + const res = await fetch(`${config.apiBase}/chat/warm`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + tabId, + ...(params?.keyId ? { keyId: params.keyId } : {}), + ...(params?.modelId ? { modelId: params.modelId } : {}), + ...(params?.agentModels ? { agentModels: params.agentModels } : {}), + }), + }); + // A newer cancel/fire superseded this request — drop its result so it + // can't clobber fresher state (e.g. user sent a real message meanwhile). + if (fireTokens.get(tabId) !== token) return; + + if (!res.ok) { + let msg = `warm failed (HTTP ${res.status})`; + try { + const body = (await res.json()) as { error?: string }; + if (body?.error) msg = body.error; + } catch { + /* non-JSON error body — keep the HTTP status message */ + } + s.error = msg; + } else { + const data = (await res.json()) as { + usage?: { inputTokens?: number; cacheReadTokens?: number }; + }; + const u = data.usage ?? {}; + s.lastPct = computeCachePct(u.inputTokens ?? 0, u.cacheReadTokens ?? 0); + s.error = null; + } + } catch (err) { + if (fireTokens.get(tabId) !== token) return; + s.error = err instanceof Error ? err.message : String(err); + } finally { + if (fireTokens.get(tabId) === token) { + s.firing = false; + // Re-arm for the next cycle (resets the 4-min countdown), but only + // if still enabled and the tab is still idle. + if (s.enabled && !runningTabs.has(tabId)) arm(tabId); + else if (!anyArmed()) stopTicker(); + } + } + } + + // ─── Public lifecycle hooks (called by the tab store) ──────────── + + /** + * Register the resolver the store uses to fetch a tab's request params + * (key/model/agentModels) at fire time. Called once by the tab store. + */ + function setRequestResolver(fn: (tabId: string) => WarmRequestParams | null): void { + resolveParams = fn; + } + + /** Seed a tab's state from persistence. Arms immediately if enabled+idle. */ + function initTab(tabId: string): void { + const s = ensure(tabId); + if (s.enabled && !runningTabs.has(tabId) && s.nextFireAt === null) { + arm(tabId); + } + } + + /** Toggle warming for a tab (persisted). Arms or cancels accordingly. */ + function setEnabled(tabId: string, enabled: boolean): void { + const s = ensure(tabId); + s.enabled = enabled; + saveCacheWarmEnabled(tabId, enabled); + if (enabled) arm(tabId); + else cancel(tabId); + } + + /** A turn started / generation is active — never warm during a turn. */ + function onTurnActive(tabId: string): void { + runningTabs.add(tabId); + cancel(tabId); + } + + /** A turn ended (tab idle) — re-arm the 4-minute countdown if enabled. */ + function onTurnEnded(tabId: string): void { + runningTabs.delete(tabId); + const s = ensure(tabId); + if (s.enabled) arm(tabId); + } + + /** + * The user sent a real message — disable+reset the timer immediately. The + * turn this message starts will re-arm warming via `onTurnEnded` once it + * settles, so the real message lands on a cache with no throwaway turns. + */ + function onUserMessage(tabId: string): void { + cancel(tabId); + } + + /** Forget a closed tab's timers/state. */ + function removeTab(tabId: string): void { + cancel(tabId); + fireTimers.delete(tabId); + fireTokens.delete(tabId); + runningTabs.delete(tabId); + delete states[tabId]; + if (!anyArmed()) stopTicker(); + } + + /** + * Forget a tab AND drop its persisted preference — for an explicit user + * close/archive. (`removeTab` keeps the persisted flag so an ephemeral + * idle-cleanup or a later reopen restores the user's choice.) + */ + function forgetTab(tabId: string): void { + removeTab(tabId); + clearCacheWarmEnabled(tabId); + } + + /** Reactive state for a tab (creates a default-off entry if absent). */ + function stateFor(tabId: string | null | undefined): WarmState { + if (!tabId) return defaultState(false); + return ensure(tabId); + } + + return { + setRequestResolver, + initTab, + setEnabled, + onTurnActive, + onTurnEnded, + onUserMessage, + removeTab, + forgetTab, + stateFor, + /** Reactive ticking clock (epoch ms) for countdown rendering. */ + get now() { + return now; + }, + // Exposed for tests to drive a fire without waiting 4 minutes. + fireNow: fire, + }; +} + +export const cacheWarming = createCacheWarmingStore(); diff --git a/packages/frontend/src/lib/components/ModelSelector.svelte b/packages/frontend/src/lib/components/ModelSelector.svelte index 8601795..1e6cdf0 100644 --- a/packages/frontend/src/lib/components/ModelSelector.svelte +++ b/packages/frontend/src/lib/components/ModelSelector.svelte @@ -10,8 +10,13 @@ const modelCache = new Map<string, string[]>(); REASONING_EFFORT_LABELS, } from "@dispatch/core/src/types/index.js"; import type { KeyInfo } from "../types.js"; + import { + cacheWarming, + WARM_INTERVAL_MS, + } from "../cache-warming.svelte.js"; import { config } from "../config.js"; import { router } from "../router.svelte.js"; + import { tabStore } from "../tabs.svelte.js"; interface AgentInfo { name: string; @@ -51,6 +56,7 @@ const modelCache = new Map<string, string[]>(); const { keys = [], + activeTabId = null, activeKeyId = null, activeModelId = null, reasoningEffort = "max", @@ -65,6 +71,7 @@ const modelCache = new Map<string, string[]>(); onWorkingDirectoryChange = (_dir: string | null) => {}, }: { keys?: KeyInfo[]; + activeTabId?: string | null; activeKeyId?: string | null; activeModelId?: string | null; reasoningEffort?: string; @@ -87,6 +94,26 @@ const modelCache = new Map<string, string[]>(); let sliderDragging = $state<number | null>(null); let modelSearch = $state(""); + // ─── Prompt-cache warming (debug strip lives at the bottom) ────── + // Reactive per-tab warming state from the singleton store. `warm.now` is a + // 1s ticking clock so the countdown re-renders while a fire is pending. + const warm = $derived(cacheWarming.stateFor(activeTabId)); + const warmCountdown = $derived.by(() => { + const next = warm.nextFireAt; + if (next === null) return null; + const ms = Math.max(0, next - cacheWarming.now); + const total = Math.round(ms / 1000); + const m = Math.floor(total / 60); + const s = total % 60; + return `${m}:${s.toString().padStart(2, "0")}`; + }); + const warmIntervalLabel = `${Math.round(WARM_INTERVAL_MS / 60000)} min`; + + function toggleCacheWarming(enabled: boolean): void { + if (!activeTabId) return; + tabStore.setCacheWarmingEnabled(activeTabId, enabled); + } + let cwdExists = $state<boolean | null>(null); let cwdCheckTimer: ReturnType<typeof setTimeout> | null = null; @@ -434,6 +461,68 @@ const modelCache = new Map<string, string[]>(); Agent Settings </button> {/if} + + <!-- Prompt-cache warming (bottom of the Chat Settings panel) --> + <div class="mt-3 pt-3 border-t border-base-300"> + <label class="flex items-center gap-2 cursor-pointer"> + <input + type="checkbox" + class="checkbox checkbox-sm rounded-sm" + checked={warm.enabled} + disabled={!activeTabId} + onchange={(e) => toggleCacheWarming(e.currentTarget.checked)} + /> + <span class="text-xs font-semibold">Keep prompt cache warm</span> + </label> + <p class="text-[10px] text-base-content/40 mt-1 leading-snug"> + While this tab is idle, replays the cached conversation every {warmIntervalLabel} + so the provider cache stays warm for your next message. Warming traffic is + debug-only — it never touches history, the Cache Rate metric, or context size. + </p> + + {#if warm.enabled} + <div class="mt-2 flex flex-col gap-2 bg-base-300/40 rounded-lg p-2"> + <!-- Warming "last request" cache rate (separate from the real metric) --> + <div class="flex flex-col gap-0.5"> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50">Last request (warming)</span> + <span class="text-xs font-mono">{warm.lastPct === null ? "-%" : `${warm.lastPct}%`}</span> + </div> + <progress + class="progress w-full h-2 {warm.lastPct === null + ? '' + : warm.lastPct >= 70 + ? 'progress-success' + : warm.lastPct >= 30 + ? 'progress-warning' + : 'progress-error'}" + value={warm.lastPct ?? 0} + max="100" + ></progress> + </div> + + <!-- Countdown to the next warming fire --> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50">Next warm in</span> + <span class="text-xs font-mono"> + {#if warm.firing} + warming… + {:else if warmCountdown !== null} + {warmCountdown} + {:else} + — + {/if} + </span> + </div> + + {#if warm.error} + <div class="text-[10px] text-error break-words"> + {warm.error} + </div> + {/if} + </div> + {/if} + </div> </div> {#if showKeyModal} diff --git a/packages/frontend/src/lib/components/SidebarPanel.svelte b/packages/frontend/src/lib/components/SidebarPanel.svelte index 519f411..ff8556a 100644 --- a/packages/frontend/src/lib/components/SidebarPanel.svelte +++ b/packages/frontend/src/lib/components/SidebarPanel.svelte @@ -31,6 +31,7 @@ const { contextLimit = null, permissionLog = [], apiBase = "", + activeTabId = null as string | null, activeKeyId = null, activeModelId = null, reasoningEffort = "max", @@ -52,6 +53,7 @@ const { contextLimit?: number | null; permissionLog?: LogEntry[]; apiBase?: string; + activeTabId?: string | null; activeKeyId?: string | null; activeModelId?: string | null; reasoningEffort?: string; @@ -157,6 +159,7 @@ function contentClass(_selected: string): string { {#if panel.selected === "Chat Settings"} <ModelSelector {keys} + {activeTabId} {activeKeyId} {activeModelId} {reasoningEffort} diff --git a/packages/frontend/src/lib/tabs.svelte.ts b/packages/frontend/src/lib/tabs.svelte.ts index e33a0e9..a0125ef 100644 --- a/packages/frontend/src/lib/tabs.svelte.ts +++ b/packages/frontend/src/lib/tabs.svelte.ts @@ -19,6 +19,7 @@ import { type ReasoningEffort, } from "@dispatch/core/src/types/index.js"; import { intactTokenIds, type StagedAttachment } from "./attachment-tokens.js"; +import { cacheWarming } from "./cache-warming.svelte.js"; import { config } from "./config.js"; import { appSettings } from "./settings.svelte.js"; import type { @@ -245,6 +246,14 @@ export function createTabStore() { handleEvent(event as AgentEvent & { tabId?: string }); }); + // Let the cache-warming store resolve a tab's provider request params + // (key/model/fallback chain) at fire time, straight from live tab state. + cacheWarming.setRequestResolver((tabId) => { + const t = getTabById(tabId); + if (!t) return null; + return { keyId: t.keyId, modelId: t.modelId, agentModels: t.agentModels }; + }); + $effect.root(() => { $effect(() => { isConnected = wsClient.connectionStatus === "connected"; @@ -327,6 +336,7 @@ export function createTabStore() { }; tabs = [...tabs, tab]; activeTabId = id; + cacheWarming.initTab(id); // Auto-check default skills then apply default agent (sequential to avoid race) void (async () => { @@ -405,6 +415,7 @@ export function createTabStore() { }; tabs = [...tabs, newTab]; activeTabId = agentId; + cacheWarming.initTab(agentId); evictChunks(agentId); } catch (err) { console.error("openAgentTab failed:", err); @@ -415,6 +426,8 @@ export function createTabStore() { const tab = getTabById(id); if (!tab) return; + cacheWarming.forgetTab(id); + // Archive on backend (also stops any running agent) try { await fetch(`${config.apiBase}/tabs/${id}`, { method: "DELETE" }); @@ -974,6 +987,11 @@ export function createTabStore() { // Trim each restored tab down to the chunk limit (user starts at bottom). for (const t of restored) { evictChunks(t.id); + // Seed warming from persisted per-tab preference. Arms the 4-minute + // countdown for idle+enabled tabs; running tabs stay paused until + // their next `status`/`statuses` reconcile flips them idle. + cacheWarming.initTab(t.id); + if (t.agentStatus === "running") cacheWarming.onTurnActive(t.id); } // Activate the first restored tab (the list is already ordered by // `position` from the backend). @@ -988,6 +1006,11 @@ export function createTabStore() { case "status": { if (!tabId) break; updateTab(tabId, { agentStatus: event.status }); + // Cache warming never fires mid-turn: pause it while running, and + // re-arm the 4-minute countdown once the turn ends (idle/error). + if (event.status === "running") { + cacheWarming.onTurnActive(tabId); + } if (event.status === "idle" || event.status === "error") { // Stop the streaming cursor immediately; the fold of the live // tail into the sealed chunk log happens on `turn-sealed` @@ -998,7 +1021,10 @@ export function createTabStore() { updateTab(tabId, { currentAssistantId: null }); const tab = getTabById(tabId); if (tab && !tab.persistent && tabId !== activeTabId) { + cacheWarming.removeTab(tabId); tabs = tabs.filter((t) => t.id !== tabId); + } else { + cacheWarming.onTurnEnded(tabId); } } break; @@ -1084,6 +1110,14 @@ export function createTabStore() { updateTab(t.id, { agentStatus: backendStatus }); } + // Sync cache warming to the reconciled status: pause it while a + // tab is (still) running, otherwise (re-)arm the idle countdown. + if (backendStatus === "running") { + cacheWarming.onTurnActive(t.id); + } else { + cacheWarming.onTurnEnded(t.id); + } + // Rehydrate the todo list from the snapshot (backend truth) // so a reconnect/reload doesn't blank the Tasks panel. updateTab(t.id, { tasks: snap?.tasks ?? [] }); @@ -1643,6 +1677,12 @@ export function createTabStore() { let tab = getActiveTab(); if (!tab) return; + // A real user message disables+resets the warming timer immediately, so + // the genuine turn appends to the real history with NO throwaway turns + // present (it lands on the warm cache). Warming re-arms when this turn + // ends (see the `status` handler → cacheWarming.onTurnEnded). + cacheWarming.onUserMessage(tab.id); + // Refresh agent config to pick up any changes made in AgentBuilder if (tab.agentSlug && tab.agentScope) { await refreshAgentConfig(tab.id); @@ -1902,6 +1942,14 @@ export function createTabStore() { updateTab(tab.id, { workingDirectory: dir || null }); } + /** + * Enable/disable prompt-cache warming for a tab (persisted per-tab). The + * warming store arms or cancels its 4-minute idle timer accordingly. + */ + function setCacheWarmingEnabled(tabId: string, enabled: boolean): void { + cacheWarming.setEnabled(tabId, enabled); + } + function setAgent( agent: { slug: string; @@ -2176,6 +2224,7 @@ export function createTabStore() { promoteTab, openAgentTab, setWorkingDirectory, + setCacheWarmingEnabled, // Exposed so tests can drive the real reactive code path that the // WS callback uses in production. Not intended for use in // components — they should rely on the WS subscription instead. diff --git a/packages/frontend/tests/cache-warm-storage.test.ts b/packages/frontend/tests/cache-warm-storage.test.ts new file mode 100644 index 0000000..458d07d --- /dev/null +++ b/packages/frontend/tests/cache-warm-storage.test.ts @@ -0,0 +1,111 @@ +/** + * Tests for `src/lib/cache-warm-storage.ts` — the per-tab localStorage + * round-trip for the "keep prompt cache warm" toggle. + * + * As in `sidebar-storage.test.ts`, Bun's `localStorage` shim is partial, so we + * install a clean in-memory polyfill per-test. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + clearCacheWarmEnabled, + loadCacheWarmEnabled, + saveCacheWarmEnabled, +} from "../src/lib/cache-warm-storage.js"; + +const LS_KEY = "dispatch-cache-warming"; + +function makeLocalStorageMock(): Storage { + const store = new Map<string, string>(); + return { + getItem: (k: string) => store.get(k) ?? null, + setItem: (k: string, v: string) => { + store.set(k, v); + }, + removeItem: (k: string) => { + store.delete(k); + }, + clear: () => { + store.clear(); + }, + get length() { + return store.size; + }, + key: (i: number) => Array.from(store.keys())[i] ?? null, + }; +} + +beforeEach(() => { + vi.stubGlobal("localStorage", makeLocalStorageMock()); +}); + +describe("loadCacheWarmEnabled", () => { + it("defaults to false when nothing is stored", () => { + expect(loadCacheWarmEnabled("tab-1")).toBe(false); + }); + + it("returns true after enabling a tab", () => { + saveCacheWarmEnabled("tab-1", true); + expect(loadCacheWarmEnabled("tab-1")).toBe(true); + }); + + it("is scoped per-tab (one tab's flag doesn't leak to another)", () => { + saveCacheWarmEnabled("tab-1", true); + expect(loadCacheWarmEnabled("tab-2")).toBe(false); + }); + + it("returns false when the stored JSON is malformed", () => { + localStorage.setItem(LS_KEY, "not json {["); + expect(loadCacheWarmEnabled("tab-1")).toBe(false); + }); + + it("ignores non-boolean entries", () => { + localStorage.setItem(LS_KEY, JSON.stringify({ "tab-1": "yes", "tab-2": true })); + expect(loadCacheWarmEnabled("tab-1")).toBe(false); + expect(loadCacheWarmEnabled("tab-2")).toBe(true); + }); +}); + +describe("saveCacheWarmEnabled", () => { + it("removes the entry when set to false (default-off representation)", () => { + saveCacheWarmEnabled("tab-1", true); + saveCacheWarmEnabled("tab-1", false); + const raw = localStorage.getItem(LS_KEY); + expect(raw).toBe(JSON.stringify({})); + expect(loadCacheWarmEnabled("tab-1")).toBe(false); + }); + + it("preserves other tabs' flags when toggling one", () => { + saveCacheWarmEnabled("tab-1", true); + saveCacheWarmEnabled("tab-2", true); + saveCacheWarmEnabled("tab-1", false); + expect(loadCacheWarmEnabled("tab-1")).toBe(false); + expect(loadCacheWarmEnabled("tab-2")).toBe(true); + }); + + it("silently ignores storage write errors", () => { + vi.stubGlobal("localStorage", { + getItem: () => null, + setItem: () => { + throw new Error("QuotaExceededError"); + }, + removeItem: () => {}, + clear: () => {}, + length: 0, + key: () => null, + }); + expect(() => saveCacheWarmEnabled("tab-1", true)).not.toThrow(); + }); +}); + +describe("clearCacheWarmEnabled", () => { + it("drops a tab's persisted flag", () => { + saveCacheWarmEnabled("tab-1", true); + clearCacheWarmEnabled("tab-1"); + expect(loadCacheWarmEnabled("tab-1")).toBe(false); + }); + + it("is a no-op when the tab has no stored flag", () => { + expect(() => clearCacheWarmEnabled("tab-x")).not.toThrow(); + expect(localStorage.getItem(LS_KEY)).toBeNull(); + }); +}); diff --git a/packages/frontend/tests/cache-warming.test.ts b/packages/frontend/tests/cache-warming.test.ts new file mode 100644 index 0000000..012efb1 --- /dev/null +++ b/packages/frontend/tests/cache-warming.test.ts @@ -0,0 +1,219 @@ +/** + * Tests for `src/lib/cache-warming.svelte.ts` — the prompt-cache warming + * timer/orchestration store. + * + * Drives the REAL `$state`-backed store via `createCacheWarmingStore()` under + * fake timers, asserting the idle/turn/send lifecycle, the repeating 4-minute + * cadence, the warming-specific last-% capture, and error surfacing. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// `config.ts` reads localStorage at import; mock it (mirrors chat-store.test). +vi.mock("../src/lib/config.js", () => ({ + config: { + apiBase: "http://test.local:3000", + wsUrl: "ws://test.local:3000/ws", + defaultApiBase: "http://test.local:3000", + setApiBase: vi.fn(), + }, +})); + +function makeLocalStorageMock(): Storage { + const store = new Map<string, string>(); + return { + getItem: (k: string) => store.get(k) ?? null, + setItem: (k: string, v: string) => { + store.set(k, v); + }, + removeItem: (k: string) => { + store.delete(k); + }, + clear: () => { + store.clear(); + }, + get length() { + return store.size; + }, + key: (i: number) => Array.from(store.keys())[i] ?? null, + }; +} + +import { createCacheWarmingStore, WARM_INTERVAL_MS } from "../src/lib/cache-warming.svelte.js"; + +type WarmStore = ReturnType<typeof createCacheWarmingStore>; + +function makeFetchOk(usage: { inputTokens: number; cacheReadTokens: number }): typeof fetch { + return vi.fn(() => + Promise.resolve({ + ok: true, + status: 200, + json: () => Promise.resolve({ usage }), + }), + ) as unknown as typeof fetch; +} + +/** Flush microtasks so awaited fetch `.json()` chains settle under fake timers. */ +async function flush(): Promise<void> { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +let store: WarmStore; + +beforeEach(() => { + vi.useFakeTimers(); + vi.stubGlobal("localStorage", makeLocalStorageMock()); + store = createCacheWarmingStore(); +}); + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +describe("enable / disable lifecycle", () => { + it("arms a countdown when enabled and idle", () => { + store.setEnabled("tab-1", true); + const s = store.stateFor("tab-1"); + expect(s.enabled).toBe(true); + expect(s.nextFireAt).not.toBeNull(); + }); + + it("cancels the countdown when disabled", () => { + store.setEnabled("tab-1", true); + store.setEnabled("tab-1", false); + expect(store.stateFor("tab-1").nextFireAt).toBeNull(); + }); + + it("does not arm a disabled tab", () => { + const s = store.stateFor("tab-2"); + expect(s.enabled).toBe(false); + expect(s.nextFireAt).toBeNull(); + }); +}); + +describe("firing cadence", () => { + it("fires after the interval and captures the warming last-%", async () => { + const fetchMock = makeFetchOk({ inputTokens: 1000, cacheReadTokens: 900 }); + vi.stubGlobal("fetch", fetchMock); + + store.setRequestResolver(() => ({ keyId: "k", modelId: "m", agentModels: null })); + store.setEnabled("tab-1", true); + + await vi.advanceTimersByTimeAsync(WARM_INTERVAL_MS); + await flush(); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, opts] = (fetchMock as unknown as { mock: { calls: unknown[][] } }).mock + .calls[0] as [string, { body: string }]; + expect(url).toContain("/chat/warm"); + expect(JSON.parse(opts.body)).toMatchObject({ tabId: "tab-1", keyId: "k", modelId: "m" }); + + const s = store.stateFor("tab-1"); + expect(s.lastPct).toBe(90); // 900 / 1000 + expect(s.error).toBeNull(); + }); + + it("re-arms repeatedly (not a one-shot)", async () => { + const fetchMock = makeFetchOk({ inputTokens: 1000, cacheReadTokens: 800 }); + vi.stubGlobal("fetch", fetchMock); + store.setEnabled("tab-1", true); + + await vi.advanceTimersByTimeAsync(WARM_INTERVAL_MS); + await flush(); + expect(fetchMock).toHaveBeenCalledTimes(1); + // After the first fire it re-arms, so a second interval fires again. + expect(store.stateFor("tab-1").nextFireAt).not.toBeNull(); + + await vi.advanceTimersByTimeAsync(WARM_INTERVAL_MS); + await flush(); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("shows '-%' (null) before it has ever fired", () => { + store.setEnabled("tab-1", true); + expect(store.stateFor("tab-1").lastPct).toBeNull(); + }); +}); + +describe("turn-state gating", () => { + it("does NOT fire while a turn is active", async () => { + const fetchMock = makeFetchOk({ inputTokens: 10, cacheReadTokens: 5 }); + vi.stubGlobal("fetch", fetchMock); + store.setEnabled("tab-1", true); + store.onTurnActive("tab-1"); + expect(store.stateFor("tab-1").nextFireAt).toBeNull(); + + await vi.advanceTimersByTimeAsync(WARM_INTERVAL_MS * 2); + await flush(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("re-arms when the turn ends (idle)", () => { + store.setEnabled("tab-1", true); + store.onTurnActive("tab-1"); + store.onTurnEnded("tab-1"); + expect(store.stateFor("tab-1").nextFireAt).not.toBeNull(); + }); + + it("a real user message disables+resets the pending timer", async () => { + const fetchMock = makeFetchOk({ inputTokens: 10, cacheReadTokens: 5 }); + vi.stubGlobal("fetch", fetchMock); + store.setEnabled("tab-1", true); + store.onUserMessage("tab-1"); + expect(store.stateFor("tab-1").nextFireAt).toBeNull(); + + await vi.advanceTimersByTimeAsync(WARM_INTERVAL_MS); + await flush(); + // The pending fire was cancelled by onUserMessage; nothing fired. + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe("error surfacing", () => { + it("captures a non-OK HTTP error into state.error", async () => { + const fetchMock = vi.fn(() => + Promise.resolve({ + ok: false, + status: 500, + json: () => Promise.resolve({ error: "provider exploded" }), + }), + ) as unknown as typeof fetch; + vi.stubGlobal("fetch", fetchMock); + + store.setEnabled("tab-1", true); + await vi.advanceTimersByTimeAsync(WARM_INTERVAL_MS); + await flush(); + + expect(store.stateFor("tab-1").error).toBe("provider exploded"); + }); + + it("captures a network rejection into state.error", async () => { + const fetchMock = vi.fn(() => + Promise.reject(new Error("network down")), + ) as unknown as typeof fetch; + vi.stubGlobal("fetch", fetchMock); + + store.setEnabled("tab-1", true); + await vi.advanceTimersByTimeAsync(WARM_INTERVAL_MS); + await flush(); + + expect(store.stateFor("tab-1").error).toBe("network down"); + }); +}); + +describe("forgetTab / removeTab", () => { + it("forgetTab clears state, timers, and persisted preference for a closed tab", async () => { + const fetchMock = makeFetchOk({ inputTokens: 10, cacheReadTokens: 5 }); + vi.stubGlobal("fetch", fetchMock); + store.setEnabled("tab-1", true); + store.forgetTab("tab-1"); + + // A fresh stateFor after removal is a default-off entry. + expect(store.stateFor("tab-1").enabled).toBe(false); + await vi.advanceTimersByTimeAsync(WARM_INTERVAL_MS * 2); + await flush(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); |
