diff options
| author | Adam Malczewski <[email protected]> | 2026-06-03 13:02:15 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-03 13:02:15 +0900 |
| commit | e87e6b39285c8001045d1ebdac873b182c0f7868 (patch) | |
| tree | 27003852f7b182fd65c6ad762784aa5fcf839ebc /packages/api/tests | |
| parent | ae672fd4f5542a2c217cf97657bf81eeebdaabbd (diff) | |
| download | dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.tar.gz dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.zip | |
feat: prompt cache warming for idle tabs
Keep a tab's provider prompt-cache warm while idle by periodically replaying
the exact cached conversation prefix plus a single trivial throwaway turn,
resetting the provider's ~5-min cache TTL so the user's next real message hits
a warm cache.
Backend:
- Agent.warmCache(history): extracts buildLlmContext() shared with run(), then
re-sends the identical system+tools+history prefix (same Anthropic
cache_control breakpoints) plus a 'reply with just a .' probe turn via
toolChoice:none. Returns the request usage; mutates no history, emits/persists
nothing.
- AgentManager.warmCacheForTab(): resolves the same agent the next real turn
would use, replays the FULL genuine history, refuses while a turn is running.
- POST /chat/warm: returns ONLY the warming request's usage (never persisted,
never folded into the real usage aggregate).
Frontend:
- cache-warming.svelte.ts store: per-tab 4-min repeating idle timer with
countdown, warming-specific last-request cache %, and error capture. Arms on
turn end, pauses during a turn, disables+resets on a real user message.
- cache-warm-storage.ts: per-tab localStorage persistence of the toggle.
- Lifecycle hooks wired into tabs.svelte.ts (status/statuses/sendMessage/
hydrate/create/open/close).
- ModelSelector: bottom-of-panel checkbox + debug strip (last-% / countdown /
error), shown only when enabled. Warming cache data never touches the real
Cache Rate view.
Tests: core warmCache (5), api warm route (3) + warmCacheForTab (3), frontend
store (12) + storage (10). check / test (779) / frontend build / typecheck all
green.
Diffstat (limited to 'packages/api/tests')
| -rw-r--r-- | packages/api/tests/agent-manager.test.ts | 64 | ||||
| -rw-r--r-- | packages/api/tests/routes.test.ts | 53 |
2 files changed, 117 insertions, 0 deletions
diff --git a/packages/api/tests/agent-manager.test.ts b/packages/api/tests/agent-manager.test.ts index 788106e..707039a 100644 --- a/packages/api/tests/agent-manager.test.ts +++ b/packages/api/tests/agent-manager.test.ts @@ -91,6 +91,13 @@ function resetCapturedRunOptions(): void { capturedRunOptions.length = 0; } +// Capture every warmCache(history) call so tests can assert the warming replay +// receives the genuine (FULL) history and returns its usage unmodified. +const capturedWarmHistories: unknown[][] = []; +function resetCapturedWarmHistories(): void { + capturedWarmHistories.length = 0; +} + // Configurable settings store so tests can toggle tool permissions // (perm_send_to_tab / perm_read_tab / ...) and assert which tools the // constructed Agent receives. Defaults to empty (getSetting → null). @@ -185,6 +192,15 @@ vi.mock("@dispatch/core", () => ({ } for await (const ev of defaultRun(message)) yield ev; } + async warmCache(history: unknown[]) { + capturedWarmHistories.push([...history]); + return { + inputTokens: 1200, + outputTokens: 1, + cacheReadTokens: 1100, + cacheWriteTokens: 0, + }; + } }, PermissionService: class MockPermissionService { ask(_request: unknown, _rulesets: unknown[]) { @@ -512,6 +528,7 @@ describe("AgentManager", () => { appendEventToChunksSpy.mockClear(); resetAppendChunksCalls(); resetFakeUsageStats(); + resetCapturedWarmHistories(); }); it("initial status is idle", () => { @@ -1858,4 +1875,51 @@ describe("AgentManager", () => { }); }); }); + + describe("warmCacheForTab (prompt-cache warming)", () => { + it("returns the warm request usage and forwards the FULL genuine history", async () => { + const manager = new AgentManager(); + setFakeMessages("tab-warm", [ + makeRow("tab-warm", 1, "user", [{ type: "text", text: "hello" }]), + makeRow("tab-warm", 2, "assistant", [{ type: "text", text: "hi" }]), + ]); + + const result = await manager.warmCacheForTab("tab-warm"); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.usage).toEqual({ + inputTokens: 1200, + outputTokens: 1, + cacheReadTokens: 1100, + cacheWriteTokens: 0, + }); + } + // The genuine history is forwarded UNTRIMMED (both turns), so the + // replayed prefix matches the next real turn exactly. + expect(capturedWarmHistories).toHaveLength(1); + expect(capturedWarmHistories[0]).toHaveLength(2); + }); + + it("does NOT persist anything (no appendChunks for the warm request)", async () => { + const manager = new AgentManager(); + setFakeMessages("tab-warm-2", [ + makeRow("tab-warm-2", 1, "user", [{ type: "text", text: "hello" }]), + ]); + await manager.warmCacheForTab("tab-warm-2"); + // Warming must never write chunk rows (history / usage / anything). + expect(appendChunksCalls).toHaveLength(0); + }); + + it("refuses to warm while the tab is generating", async () => { + const manager = new AgentManager(); + // Start a turn (status flips to running) but don't await it. + const running = manager.processMessage("tab-warm-busy", "go"); + // Let the mock run() yield its first running status. + await new Promise<void>((r) => setTimeout(r, 1)); + const result = await manager.warmCacheForTab("tab-warm-busy"); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toBe("tab is generating"); + await running; + }); + }); }); diff --git a/packages/api/tests/routes.test.ts b/packages/api/tests/routes.test.ts index 7cfd8a7..d2c8229 100644 --- a/packages/api/tests/routes.test.ts +++ b/packages/api/tests/routes.test.ts @@ -24,6 +24,15 @@ vi.mock("@dispatch/core", () => ({ Agent: class MockAgent { status = "idle"; messages: unknown[] = []; + async warmCache(_history: unknown[]) { + // Simulate a warm replay that read most of the prompt from cache. + return { + inputTokens: 1000, + outputTokens: 1, + cacheReadTokens: 900, + cacheWriteTokens: 0, + }; + } async *run(_message: string) { yield { type: "status", status: "running" } as const; // Simulate some processing time so status stays "running" @@ -665,6 +674,50 @@ describe("POST /chat/stop", () => { expect(res.status).toBe(400); }); }); + +describe("POST /chat/warm", () => { + it("returns ONLY the warming request usage (never persisted/emitted)", async () => { + const res = await app.request("/chat/warm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ tabId: "tab-warm-1" }), + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { usage?: Record<string, number> }; + expect(body.usage).toEqual({ + inputTokens: 1000, + outputTokens: 1, + cacheReadTokens: 900, + cacheWriteTokens: 0, + }); + }); + + it("returns 400 when tabId is missing", async () => { + const res = await app.request("/chat/warm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + expect(res.status).toBe(400); + }); + + it("returns 409 while the tab is generating", async () => { + // Kick off a real (mock) turn so the tab is "running", then immediately + // attempt to warm it — warming must refuse mid-turn. + await app.request("/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ tabId: "tab-warm-busy", message: "hi" }), + }); + const res = await app.request("/chat/warm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ tabId: "tab-warm-busy" }), + }); + expect(res.status).toBe(409); + }); +}); + describe("Wake schedule routes", () => { async function getSchedule() { const res = await app.request("/models/wake-schedule"); |
