summaryrefslogtreecommitdiffhomepage
path: root/packages/api/tests
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-03 13:02:15 +0900
committerAdam Malczewski <[email protected]>2026-06-03 13:02:15 +0900
commite87e6b39285c8001045d1ebdac873b182c0f7868 (patch)
tree27003852f7b182fd65c6ad762784aa5fcf839ebc /packages/api/tests
parentae672fd4f5542a2c217cf97657bf81eeebdaabbd (diff)
downloaddispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.tar.gz
dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.zip
feat: prompt cache warming for idle tabs
Keep a tab's provider prompt-cache warm while idle by periodically replaying the exact cached conversation prefix plus a single trivial throwaway turn, resetting the provider's ~5-min cache TTL so the user's next real message hits a warm cache. Backend: - Agent.warmCache(history): extracts buildLlmContext() shared with run(), then re-sends the identical system+tools+history prefix (same Anthropic cache_control breakpoints) plus a 'reply with just a .' probe turn via toolChoice:none. Returns the request usage; mutates no history, emits/persists nothing. - AgentManager.warmCacheForTab(): resolves the same agent the next real turn would use, replays the FULL genuine history, refuses while a turn is running. - POST /chat/warm: returns ONLY the warming request's usage (never persisted, never folded into the real usage aggregate). Frontend: - cache-warming.svelte.ts store: per-tab 4-min repeating idle timer with countdown, warming-specific last-request cache %, and error capture. Arms on turn end, pauses during a turn, disables+resets on a real user message. - cache-warm-storage.ts: per-tab localStorage persistence of the toggle. - Lifecycle hooks wired into tabs.svelte.ts (status/statuses/sendMessage/ hydrate/create/open/close). - ModelSelector: bottom-of-panel checkbox + debug strip (last-% / countdown / error), shown only when enabled. Warming cache data never touches the real Cache Rate view. Tests: core warmCache (5), api warm route (3) + warmCacheForTab (3), frontend store (12) + storage (10). check / test (779) / frontend build / typecheck all green.
Diffstat (limited to 'packages/api/tests')
-rw-r--r--packages/api/tests/agent-manager.test.ts64
-rw-r--r--packages/api/tests/routes.test.ts53
2 files changed, 117 insertions, 0 deletions
diff --git a/packages/api/tests/agent-manager.test.ts b/packages/api/tests/agent-manager.test.ts
index 788106e..707039a 100644
--- a/packages/api/tests/agent-manager.test.ts
+++ b/packages/api/tests/agent-manager.test.ts
@@ -91,6 +91,13 @@ function resetCapturedRunOptions(): void {
capturedRunOptions.length = 0;
}
+// Capture every warmCache(history) call so tests can assert the warming replay
+// receives the genuine (FULL) history and returns its usage unmodified.
+const capturedWarmHistories: unknown[][] = [];
+function resetCapturedWarmHistories(): void {
+ capturedWarmHistories.length = 0;
+}
+
// Configurable settings store so tests can toggle tool permissions
// (perm_send_to_tab / perm_read_tab / ...) and assert which tools the
// constructed Agent receives. Defaults to empty (getSetting → null).
@@ -185,6 +192,15 @@ vi.mock("@dispatch/core", () => ({
}
for await (const ev of defaultRun(message)) yield ev;
}
+ async warmCache(history: unknown[]) {
+ capturedWarmHistories.push([...history]);
+ return {
+ inputTokens: 1200,
+ outputTokens: 1,
+ cacheReadTokens: 1100,
+ cacheWriteTokens: 0,
+ };
+ }
},
PermissionService: class MockPermissionService {
ask(_request: unknown, _rulesets: unknown[]) {
@@ -512,6 +528,7 @@ describe("AgentManager", () => {
appendEventToChunksSpy.mockClear();
resetAppendChunksCalls();
resetFakeUsageStats();
+ resetCapturedWarmHistories();
});
it("initial status is idle", () => {
@@ -1858,4 +1875,51 @@ describe("AgentManager", () => {
});
});
});
+
+ describe("warmCacheForTab (prompt-cache warming)", () => {
+ it("returns the warm request usage and forwards the FULL genuine history", async () => {
+ const manager = new AgentManager();
+ setFakeMessages("tab-warm", [
+ makeRow("tab-warm", 1, "user", [{ type: "text", text: "hello" }]),
+ makeRow("tab-warm", 2, "assistant", [{ type: "text", text: "hi" }]),
+ ]);
+
+ const result = await manager.warmCacheForTab("tab-warm");
+ expect(result.ok).toBe(true);
+ if (result.ok) {
+ expect(result.usage).toEqual({
+ inputTokens: 1200,
+ outputTokens: 1,
+ cacheReadTokens: 1100,
+ cacheWriteTokens: 0,
+ });
+ }
+ // The genuine history is forwarded UNTRIMMED (both turns), so the
+ // replayed prefix matches the next real turn exactly.
+ expect(capturedWarmHistories).toHaveLength(1);
+ expect(capturedWarmHistories[0]).toHaveLength(2);
+ });
+
+ it("does NOT persist anything (no appendChunks for the warm request)", async () => {
+ const manager = new AgentManager();
+ setFakeMessages("tab-warm-2", [
+ makeRow("tab-warm-2", 1, "user", [{ type: "text", text: "hello" }]),
+ ]);
+ await manager.warmCacheForTab("tab-warm-2");
+ // Warming must never write chunk rows (history / usage / anything).
+ expect(appendChunksCalls).toHaveLength(0);
+ });
+
+ it("refuses to warm while the tab is generating", async () => {
+ const manager = new AgentManager();
+ // Start a turn (status flips to running) but don't await it.
+ const running = manager.processMessage("tab-warm-busy", "go");
+ // Let the mock run() yield its first running status.
+ await new Promise<void>((r) => setTimeout(r, 1));
+ const result = await manager.warmCacheForTab("tab-warm-busy");
+ expect(result.ok).toBe(false);
+ if (!result.ok) expect(result.error).toBe("tab is generating");
+ await running;
+ });
+ });
});
diff --git a/packages/api/tests/routes.test.ts b/packages/api/tests/routes.test.ts
index 7cfd8a7..d2c8229 100644
--- a/packages/api/tests/routes.test.ts
+++ b/packages/api/tests/routes.test.ts
@@ -24,6 +24,15 @@ vi.mock("@dispatch/core", () => ({
Agent: class MockAgent {
status = "idle";
messages: unknown[] = [];
+ async warmCache(_history: unknown[]) {
+ // Simulate a warm replay that read most of the prompt from cache.
+ return {
+ inputTokens: 1000,
+ outputTokens: 1,
+ cacheReadTokens: 900,
+ cacheWriteTokens: 0,
+ };
+ }
async *run(_message: string) {
yield { type: "status", status: "running" } as const;
// Simulate some processing time so status stays "running"
@@ -665,6 +674,50 @@ describe("POST /chat/stop", () => {
expect(res.status).toBe(400);
});
});
+
+describe("POST /chat/warm", () => {
+ it("returns ONLY the warming request usage (never persisted/emitted)", async () => {
+ const res = await app.request("/chat/warm", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ tabId: "tab-warm-1" }),
+ });
+ expect(res.status).toBe(200);
+ const body = (await res.json()) as { usage?: Record<string, number> };
+ expect(body.usage).toEqual({
+ inputTokens: 1000,
+ outputTokens: 1,
+ cacheReadTokens: 900,
+ cacheWriteTokens: 0,
+ });
+ });
+
+ it("returns 400 when tabId is missing", async () => {
+ const res = await app.request("/chat/warm", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({}),
+ });
+ expect(res.status).toBe(400);
+ });
+
+ it("returns 409 while the tab is generating", async () => {
+ // Kick off a real (mock) turn so the tab is "running", then immediately
+ // attempt to warm it — warming must refuse mid-turn.
+ await app.request("/chat", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ tabId: "tab-warm-busy", message: "hi" }),
+ });
+ const res = await app.request("/chat/warm", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ tabId: "tab-warm-busy" }),
+ });
+ expect(res.status).toBe(409);
+ });
+});
+
describe("Wake schedule routes", () => {
async function getSchedule() {
const res = await app.request("/models/wake-schedule");