diff options
| author | Adam Malczewski <[email protected]> | 2026-06-03 13:02:15 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-03 13:02:15 +0900 |
| commit | e87e6b39285c8001045d1ebdac873b182c0f7868 (patch) | |
| tree | 27003852f7b182fd65c6ad762784aa5fcf839ebc /packages/core | |
| parent | ae672fd4f5542a2c217cf97657bf81eeebdaabbd (diff) | |
| download | dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.tar.gz dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.zip | |
feat: prompt cache warming for idle tabs
Keep a tab's provider prompt-cache warm while idle by periodically replaying
the exact cached conversation prefix plus a single trivial throwaway turn,
resetting the provider's ~5-min cache TTL so the user's next real message hits
a warm cache.
Backend:
- Agent.warmCache(history): extracts buildLlmContext() shared with run(), then
re-sends the identical system+tools+history prefix (same Anthropic
cache_control breakpoints) plus a 'reply with just a .' probe turn via
toolChoice:none. Returns the request usage; mutates no history, emits/persists
nothing.
- AgentManager.warmCacheForTab(): resolves the same agent the next real turn
would use, replays the FULL genuine history, refuses while a turn is running.
- POST /chat/warm: returns ONLY the warming request's usage (never persisted,
never folded into the real usage aggregate).
Frontend:
- cache-warming.svelte.ts store: per-tab 4-min repeating idle timer with
countdown, warming-specific last-request cache %, and error capture. Arms on
turn end, pauses during a turn, disables+resets on a real user message.
- cache-warm-storage.ts: per-tab localStorage persistence of the toggle.
- Lifecycle hooks wired into tabs.svelte.ts (status/statuses/sendMessage/
hydrate/create/open/close).
- ModelSelector: bottom-of-panel checkbox + debug strip (last-% / countdown /
error), shown only when enabled. Warming cache data never touches the real
Cache Rate view.
Tests: core warmCache (5), api warm route (3) + warmCacheForTab (3), frontend
store (12) + storage (10). check / test (779) / frontend build / typecheck all
green.
Diffstat (limited to 'packages/core')
| -rw-r--r-- | packages/core/src/agent/agent.ts | bin | 60515 -> 65763 bytes | |||
| -rw-r--r-- | packages/core/tests/agent/agent.test.ts | 107 |
2 files changed, 107 insertions, 0 deletions
diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts Binary files differindex 08b317a..d0a3bb9 100644 --- a/packages/core/src/agent/agent.ts +++ b/packages/core/src/agent/agent.ts diff --git a/packages/core/tests/agent/agent.test.ts b/packages/core/tests/agent/agent.test.ts index f4b33cc..86a7a5b 100644 --- a/packages/core/tests/agent/agent.test.ts +++ b/packages/core/tests/agent/agent.test.ts @@ -1642,4 +1642,111 @@ describe("anthropicThinkingProviderOptions — adaptive-thinking model detection expect(userMsg?.content).toBe("plain text"); }); }); + + describe("warmCache (prompt-cache warming replay)", () => { + function makeWarmStream(usage: { + inputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + }) { + return makeMockStreamResult([ + { type: "text-delta", id: "t0", text: "." }, + { + type: "finish-step", + finishReason: "stop", + rawFinishReason: "stop", + usage: { + inputTokens: usage.inputTokens, + outputTokens: 1, + inputTokenDetails: { + noCacheTokens: usage.inputTokens - usage.cacheReadTokens - usage.cacheWriteTokens, + cacheReadTokens: usage.cacheReadTokens, + cacheWriteTokens: usage.cacheWriteTokens, + }, + }, + }, + finishStop, + ]); + } + + const history = [ + { role: "user" as const, chunks: [{ type: "text" as const, text: "hello" }] }, + { role: "assistant" as const, chunks: [{ type: "text" as const, text: "hi there" }] }, + ]; + + it("returns the request usage (cache read/write split) without throwing", async () => { + vi.mocked(streamText).mockReturnValue( + makeWarmStream({ inputTokens: 1000, cacheReadTokens: 950, cacheWriteTokens: 0 }), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + const usage = await agent.warmCache(history); + expect(usage).toEqual({ + inputTokens: 1000, + outputTokens: 1, + cacheReadTokens: 950, + cacheWriteTokens: 0, + }); + }); + + it("appends a single trivial throwaway user turn at the END of the history", async () => { + vi.mocked(streamText).mockReturnValue( + makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + await agent.warmCache(history); + + const callArgs = vi.mocked(streamText).mock.calls.at(-1)?.[0]; + const messages = callArgs?.messages as Array<{ role: string; content: unknown }>; + // system + 2 history messages + 1 throwaway user turn. + expect(messages[0]?.role).toBe("system"); + const last = messages.at(-1); + expect(last?.role).toBe("user"); + // The throwaway turn's text must be the trivial probe. + const lastText = JSON.stringify(last?.content); + expect(lastText).toContain("reply with just a ."); + // Exactly one extra user turn beyond the genuine history's single user msg. + const userMsgs = messages.filter((m) => m.role === "user"); + expect(userMsgs).toHaveLength(2); + }); + + it("sends Anthropic cache_control breakpoints + toolChoice none", async () => { + vi.mocked(streamText).mockReturnValue( + makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + await agent.warmCache(history); + + const callArgs = vi.mocked(streamText).mock.calls.at(-1)?.[0]; + expect(callArgs?.toolChoice).toBe("none"); + const messages = callArgs?.messages as Array<{ + role: string; + providerOptions?: { anthropic?: { cacheControl?: unknown } }; + }>; + const hasBreakpoint = messages.some( + (m) => m.providerOptions?.anthropic?.cacheControl !== undefined, + ); + expect(hasBreakpoint).toBe(true); + }); + + it("does NOT mutate the agent's own message history", async () => { + vi.mocked(streamText).mockReturnValue( + makeWarmStream({ inputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 0 }), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + expect(agent.messages).toHaveLength(0); + await agent.warmCache(history); + // warmCache takes history as an argument and never touches `this.messages`. + expect(agent.messages).toHaveLength(0); + // And it must not have flipped the agent into a running state. + expect(agent.status).toBe("idle"); + }); + + it("throws a formatted error when the stream errors", async () => { + vi.mocked(streamText).mockReturnValue( + makeMockStreamResult([{ type: "error", error: new Error("boom") }]), + ); + const agent = new Agent(makeConfig({ provider: "anthropic" })); + await expect(agent.warmCache(history)).rejects.toThrow(/boom/); + }); + }); }); |
