summaryrefslogtreecommitdiffhomepage
path: root/packages/api/src
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-03 13:02:15 +0900
committerAdam Malczewski <[email protected]>2026-06-03 13:02:15 +0900
commite87e6b39285c8001045d1ebdac873b182c0f7868 (patch)
tree27003852f7b182fd65c6ad762784aa5fcf839ebc /packages/api/src
parentae672fd4f5542a2c217cf97657bf81eeebdaabbd (diff)
downloaddispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.tar.gz
dispatch-e87e6b39285c8001045d1ebdac873b182c0f7868.zip
feat: prompt cache warming for idle tabs
Keep a tab's provider prompt-cache warm while idle by periodically replaying the exact cached conversation prefix plus a single trivial throwaway turn, resetting the provider's ~5-min cache TTL so the user's next real message hits a warm cache. Backend: - Agent.warmCache(history): extracts buildLlmContext() shared with run(), then re-sends the identical system+tools+history prefix (same Anthropic cache_control breakpoints) plus a 'reply with just a .' probe turn via toolChoice:none. Returns the request usage; mutates no history, emits/persists nothing. - AgentManager.warmCacheForTab(): resolves the same agent the next real turn would use, replays the FULL genuine history, refuses while a turn is running. - POST /chat/warm: returns ONLY the warming request's usage (never persisted, never folded into the real usage aggregate). Frontend: - cache-warming.svelte.ts store: per-tab 4-min repeating idle timer with countdown, warming-specific last-request cache %, and error capture. Arms on turn end, pauses during a turn, disables+resets on a real user message. - cache-warm-storage.ts: per-tab localStorage persistence of the toggle. - Lifecycle hooks wired into tabs.svelte.ts (status/statuses/sendMessage/ hydrate/create/open/close). - ModelSelector: bottom-of-panel checkbox + debug strip (last-% / countdown / error), shown only when enabled. Warming cache data never touches the real Cache Rate view. Tests: core warmCache (5), api warm route (3) + warmCacheForTab (3), frontend store (12) + storage (10). check / test (779) / frontend build / typecheck all green.
Diffstat (limited to 'packages/api/src')
-rw-r--r--packages/api/src/agent-manager.ts60
-rw-r--r--packages/api/src/app.ts37
2 files changed, 97 insertions, 0 deletions
diff --git a/packages/api/src/agent-manager.ts b/packages/api/src/agent-manager.ts
index c1b46b9..109dd33 100644
--- a/packages/api/src/agent-manager.ts
+++ b/packages/api/src/agent-manager.ts
@@ -8,6 +8,7 @@ import {
appendEventToChunks,
BackgroundShellStore,
BackgroundTranscriptStore,
+ type ChatMessage,
type Chunk,
type ClaudeAccount,
clearSpillForTab,
@@ -1019,6 +1020,65 @@ export class AgentManager {
}
/**
+ * Prompt-cache WARMING for an idle tab (see `Agent.warmCache`).
+ *
+ * Reconstructs the tab's genuine conversation from the persisted chunk log,
+ * resolves the SAME agent (model/key/tools/system prompt) the next real turn
+ * would use, and replays the exact cached prefix plus one trivial throwaway
+ * turn so the provider's ~5-min prompt-cache TTL is refreshed. The warming
+ * request and its response are NOT persisted, NOT emitted, and NOT folded
+ * into the real usage aggregate — its `usage` is returned to the caller so a
+ * warming-only "last request" cache rate can be shown without polluting the
+ * real Cache Rate metric.
+ *
+ * Refuses to fire while the tab is generating (`running`): the prefix would
+ * be mid-mutation and the request would contend with the live turn. Callers
+ * gate on idle anyway; this is defence in depth.
+ *
+ * Returns `{ ok: true, usage }` on success or `{ ok: false, error }` so the
+ * route can surface a debug-strip error string. Never throws.
+ */
+ async warmCacheForTab(
+ tabId: string,
+ opts: { keyId?: string; modelId?: string; agentModels?: AgentModelEntry[] } = {},
+ ): Promise<{ ok: true; usage: UsageData } | { ok: false; error: string }> {
+ if (this.getTabStatus(tabId) === "running") {
+ return { ok: false, error: "tab is generating" };
+ }
+ try {
+ const tabAgent = this._getOrCreateTabAgent(tabId);
+ if (opts.agentModels) tabAgent.agentModels = opts.agentModels;
+
+ // Resolve the agent the next REAL turn would use. The fallback chain's
+ // first entry mirrors `processMessage`'s primary attempt; we only warm
+ // the primary (warming a fallback model would write a DIFFERENT prefix).
+ const fallbackSequence = this.buildFallbackSequence(tabAgent, opts.keyId, opts.modelId);
+ const primary = fallbackSequence[0];
+ const agent = await this.getOrCreateAgentForTab(
+ tabId,
+ primary?.key_id || opts.keyId,
+ primary?.model_id || opts.modelId,
+ );
+
+ // Rebuild the genuine history exactly as `getOrCreateAgentForTab`'s
+ // pre-population does, but keep the FULL history (no trailing-user
+ // trim): warming replays the complete cached prefix as-is.
+ let history: ChatMessage[] = [];
+ try {
+ history = getMessagesForTab(tabId).map((r) => ({ role: r.role, chunks: r.chunks }));
+ } catch {
+ // DB read failed — warm with whatever in-memory history the agent has.
+ history = [...agent.messages];
+ }
+
+ const usage = await agent.warmCache(history);
+ return { ok: true, usage };
+ } catch (err) {
+ return { ok: false, error: err instanceof Error ? err.message : String(err) };
+ }
+ }
+
+ /**
* Snapshot of every tab the manager is currently tracking. Sent on WS
* connect and via GET /status so a freshly-loaded frontend can
* reconstruct any in-flight assistant turn without missing the chunks
diff --git a/packages/api/src/app.ts b/packages/api/src/app.ts
index 2f4e538..a957da7 100644
--- a/packages/api/src/app.ts
+++ b/packages/api/src/app.ts
@@ -224,6 +224,43 @@ app.post("/chat/stop", async (c) => {
return c.json({ success: true });
});
+// Prompt-cache WARMING (see AgentManager.warmCacheForTab / Agent.warmCache).
+//
+// Replays the tab's exact cached prefix + one trivial throwaway turn so the
+// provider's ~5-min prompt-cache TTL is refreshed while the tab sits idle.
+// The frontend's cache-warming timer drives this every ~4 minutes. The
+// warming request is NEVER persisted, NEVER emitted, and NEVER folded into the
+// real usage aggregate — we return ONLY its `usage` so the UI can show a
+// warming-specific "last request" cache rate without polluting the real
+// Cache Rate metric. Returns 409 when the tab is mid-turn (caller also gates).
+app.post("/chat/warm", async (c) => {
+ const body = await c.req.json<{
+ tabId?: unknown;
+ keyId?: unknown;
+ modelId?: unknown;
+ agentModels?: unknown;
+ }>();
+ const { tabId } = body;
+ if (typeof tabId !== "string" || tabId.trim() === "") {
+ return c.json({ error: "tabId must be a non-empty string" }, 400);
+ }
+ const keyId = typeof body.keyId === "string" ? body.keyId : undefined;
+ const modelId = typeof body.modelId === "string" ? body.modelId : undefined;
+ const agentModels = sanitizeAgentModels(body.agentModels);
+
+ const result = await agentManager.warmCacheForTab(tabId, {
+ ...(keyId ? { keyId } : {}),
+ ...(modelId ? { modelId } : {}),
+ ...(agentModels ? { agentModels } : {}),
+ });
+ if (!result.ok) {
+ // "tab is generating" is an expected race (not a server fault) → 409.
+ const status = result.error === "tab is generating" ? 409 : 500;
+ return c.json({ error: result.error }, status);
+ }
+ return c.json({ usage: result.usage });
+});
+
app.route("/skills", skillsRoutes);
app.route("/models", modelsRoutes);
app.route("/tabs", tabsRoutes);