summaryrefslogtreecommitdiffhomepage
path: root/packages/core
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-03 13:02:15 +0900
committerAdam Malczewski <[email protected]>2026-06-03 13:02:15 +0900
commite87e6b39285c8001045d1ebdac873b182c0f7868 (patch)
tree27003852f7b182fd65c6ad762784aa5fcf839ebc /packages/core
parentae672fd4f5542a2c217cf97657bf81eeebdaabbd (diff)
downloaddispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.tar.gz
dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.zip
feat: prompt cache warming for idle tabs
Keep a tab's provider prompt-cache warm while idle by periodically replaying the exact cached conversation prefix plus a single trivial throwaway turn, resetting the provider's ~5-min cache TTL so the user's next real message hits a warm cache. Backend: - Agent.warmCache(history): extracts buildLlmContext() shared with run(), then re-sends the identical system+tools+history prefix (same Anthropic cache_control breakpoints) plus a 'reply with just a .' probe turn via toolChoice:none. Returns the request usage; mutates no history, emits/persists nothing. - AgentManager.warmCacheForTab(): resolves the same agent the next real turn would use, replays the FULL genuine history, refuses while a turn is running. - POST /chat/warm: returns ONLY the warming request's usage (never persisted, never folded into the real usage aggregate). Frontend: - cache-warming.svelte.ts store: per-tab 4-min repeating idle timer with countdown, warming-specific last-request cache %, and error capture. Arms on turn end, pauses during a turn, disables+resets on a real user message. - cache-warm-storage.ts: per-tab localStorage persistence of the toggle. - Lifecycle hooks wired into tabs.svelte.ts (status/statuses/sendMessage/ hydrate/create/open/close). - ModelSelector: bottom-of-panel checkbox + debug strip (last-% / countdown / error), shown only when enabled. Warming cache data never touches the real Cache Rate view. Tests: core warmCache (5), api warm route (3) + warmCacheForTab (3), frontend store (12) + storage (10). check / test (779) / frontend build / typecheck all green.
Diffstat (limited to 'packages/core')
-rw-r--r--packages/core/src/agent/agent.tsbin60515 -> 65763 bytes
-rw-r--r--packages/core/tests/agent/agent.test.ts107
2 files changed, 107 insertions, 0 deletions
diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts
index 08b317a..d0a3bb9 100644
--- a/packages/core/src/agent/agent.ts
+++ b/packages/core/src/agent/agent.ts
Binary files differ
diff --git a/packages/core/tests/agent/agent.test.ts b/packages/core/tests/agent/agent.test.ts
index f4b33cc..86a7a5b 100644
--- a/packages/core/tests/agent/agent.test.ts
+++ b/packages/core/tests/agent/agent.test.ts
@@ -1642,4 +1642,111 @@ describe("anthropicThinkingProviderOptions — adaptive-thinking model detection
expect(userMsg?.content).toBe("plain text");
});
});
+
+ describe("warmCache (prompt-cache warming replay)", () => {
+ function makeWarmStream(usage: {
+ inputTokens: number;
+ cacheReadTokens: number;
+ cacheWriteTokens: number;
+ }) {
+ return makeMockStreamResult([
+ { type: "text-delta", id: "t0", text: "." },
+ {
+ type: "finish-step",
+ finishReason: "stop",
+ rawFinishReason: "stop",
+ usage: {
+ inputTokens: usage.inputTokens,
+ outputTokens: 1,
+ inputTokenDetails: {
+ noCacheTokens: usage.inputTokens - usage.cacheReadTokens - usage.cacheWriteTokens,
+ cacheReadTokens: usage.cacheReadTokens,
+ cacheWriteTokens: usage.cacheWriteTokens,
+ },
+ },
+ },
+ finishStop,
+ ]);
+ }
+
+ const history = [
+ { role: "user" as const, chunks: [{ type: "text" as const, text: "hello" }] },
+ { role: "assistant" as const, chunks: [{ type: "text" as const, text: "hi there" }] },
+ ];
+
+ it("returns the request usage (cache read/write split) without throwing", async () => {
+ vi.mocked(streamText).mockReturnValue(
+ makeWarmStream({ inputTokens: 1000, cacheReadTokens: 950, cacheWriteTokens: 0 }),
+ );
+ const agent = new Agent(makeConfig({ provider: "anthropic" }));
+ const usage = await agent.warmCache(history);
+ expect(usage).toEqual({
+ inputTokens: 1000,
+ outputTokens: 1,
+ cacheReadTokens: 950,
+ cacheWriteTokens: 0,
+ });
+ });
+
+ it("appends a single trivial throwaway user turn at the END of the history", async () => {
+ vi.mocked(streamText).mockReturnValue(
+ makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }),
+ );
+ const agent = new Agent(makeConfig({ provider: "anthropic" }));
+ await agent.warmCache(history);
+
+ const callArgs = vi.mocked(streamText).mock.calls.at(-1)?.[0];
+ const messages = callArgs?.messages as Array<{ role: string; content: unknown }>;
+ // system + 2 history messages + 1 throwaway user turn.
+ expect(messages[0]?.role).toBe("system");
+ const last = messages.at(-1);
+ expect(last?.role).toBe("user");
+ // The throwaway turn's text must be the trivial probe.
+ const lastText = JSON.stringify(last?.content);
+ expect(lastText).toContain("reply with just a .");
+ // Exactly one extra user turn beyond the genuine history's single user msg.
+ const userMsgs = messages.filter((m) => m.role === "user");
+ expect(userMsgs).toHaveLength(2);
+ });
+
+ it("sends Anthropic cache_control breakpoints + toolChoice none", async () => {
+ vi.mocked(streamText).mockReturnValue(
+ makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }),
+ );
+ const agent = new Agent(makeConfig({ provider: "anthropic" }));
+ await agent.warmCache(history);
+
+ const callArgs = vi.mocked(streamText).mock.calls.at(-1)?.[0];
+ expect(callArgs?.toolChoice).toBe("none");
+ const messages = callArgs?.messages as Array<{
+ role: string;
+ providerOptions?: { anthropic?: { cacheControl?: unknown } };
+ }>;
+ const hasBreakpoint = messages.some(
+ (m) => m.providerOptions?.anthropic?.cacheControl !== undefined,
+ );
+ expect(hasBreakpoint).toBe(true);
+ });
+
+ it("does NOT mutate the agent's own message history", async () => {
+ vi.mocked(streamText).mockReturnValue(
+ makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }),
+ );
+ const agent = new Agent(makeConfig({ provider: "anthropic" }));
+ expect(agent.messages).toHaveLength(0);
+ await agent.warmCache(history);
+ // warmCache takes history as an argument and never touches `this.messages`.
+ expect(agent.messages).toHaveLength(0);
+ // And it must not have flipped the agent into a running state.
+ expect(agent.status).toBe("idle");
+ });
+
+ it("throws a formatted error when the stream errors", async () => {
+ vi.mocked(streamText).mockReturnValue(
+ makeMockStreamResult([{ type: "error", error: new Error("boom") }]),
+ );
+ const agent = new Agent(makeConfig({ provider: "anthropic" }));
+ await expect(agent.warmCache(history)).rejects.toThrow(/boom/);
+ });
+ });
});