summaryrefslogtreecommitdiffhomepage
path: root/packages/cache-warming/src/pure.ts
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-11 14:11:13 +0900
committerAdam Malczewski <[email protected]>2026-06-11 14:11:13 +0900
commit7ffb6b28f5b6bdbfc53ebed94fc68af557612189 (patch)
treee66d9ea9d326ef771cc473d81ca5716ff78b08a8 /packages/cache-warming/src/pure.ts
parent763e5fb1c7fbfb4c7bbd43ffb935e42e5f5b5a42 (diff)
downloaddispatch-7ffb6b28f5b6bdbfc53ebed94fc68af557612189.tar.gz
dispatch-7ffb6b28f5b6bdbfc53ebed94fc68af557612189.zip
fix(cache-warming): accurate cache rate + expectedCacheRate (retention) metric
The Claude cache % read 100% whenever anything was cached, because the metric's denominator (inputTokens) excluded cached tokens on Anthropic. Fixed upstream in ../claude/provider-anthropic (inputTokens = total prompt); this commit adds the companion retention metric and exposes it: - transport-contract: WarmResponse += expectedCacheRate - transport-http: POST /chat/warm returns expectedCacheRate = cacheRead/(cacheRead+cacheWrite) - cache-warming: computeExpectedCacheRate + a per-conversation 'cache retention' surface stat - handoff: documents the fix + cache-rate vs expected-cache (cross-turn) for the FE Live-verified vs claude haiku: real turn cache rate 61% (was inflated 100%); warm within TTL expectedCacheRate=100%, after expiry=0%.
Diffstat (limited to 'packages/cache-warming/src/pure.ts')
-rw-r--r--packages/cache-warming/src/pure.ts25
1 files changed, 24 insertions, 1 deletions
diff --git a/packages/cache-warming/src/pure.ts b/packages/cache-warming/src/pure.ts
index 7b91b11..ab6fc79 100644
--- a/packages/cache-warming/src/pure.ts
+++ b/packages/cache-warming/src/pure.ts
@@ -17,6 +17,7 @@ export interface ConversationSettings {
export interface ConversationState extends ConversationSettings {
readonly active: boolean;
readonly lastPct: number | null;
+ readonly lastExpectedPct: number | null;
readonly token: number;
}
@@ -43,6 +44,21 @@ export function computeCachePct(inputTokens: number, cacheReadTokens: number): n
}
/**
+ * Compute expected cache retention rate from token counts.
+ * Of the cacheable prefix the warm touched, how much was still warm (read back)
+ * vs. had to be (re)written.
+ * Returns an integer in [0, 100]. cacheRead + cacheWrite ≤ 0 → 0.
+ */
+export function computeExpectedCacheRate(
+ cacheReadTokens: number,
+ cacheWriteTokens: number,
+): number {
+ const total = cacheReadTokens + cacheWriteTokens;
+ if (total <= 0) return 0;
+ return Math.round((cacheReadTokens / total) * 100);
+}
+
+/**
* Decide whether a conversation should be warmed right now.
* Requires: enabled, idle (not active), and the token is current (not superseded).
*/
@@ -120,8 +136,10 @@ export function buildConversationSpec(
enabled: boolean,
intervalMs: number,
lastPct: number | null,
+ lastExpectedPct: number | null,
): SurfaceSpec {
const pctDisplay = lastPct === null ? "—" : `${lastPct}%`;
+ const retentionDisplay = lastExpectedPct === null ? "—" : `${lastExpectedPct}%`;
const toggle: ToggleField = {
kind: "toggle",
label: "Enabled",
@@ -142,11 +160,16 @@ export function buildConversationSpec(
label: "Last Cache %",
value: pctDisplay,
};
+ const retentionStat: StatField = {
+ kind: "stat",
+ label: "Cache retention",
+ value: retentionDisplay,
+ };
return {
id: "cache-warming",
region: "side",
title: "Cache Warming",
- fields: [toggle, interval, stat],
+ fields: [toggle, interval, stat, retentionStat],
};
}