From 7ffb6b28f5b6bdbfc53ebed94fc68af557612189 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Thu, 11 Jun 2026 14:11:13 +0900 Subject: fix(cache-warming): accurate cache rate + expectedCacheRate (retention) metric The Claude cache % read 100% whenever anything was cached, because the metric's denominator (inputTokens) excluded cached tokens on Anthropic. Fixed upstream in ../claude/provider-anthropic (inputTokens = total prompt); this commit adds the companion retention metric and exposes it: - transport-contract: WarmResponse += expectedCacheRate - transport-http: POST /chat/warm returns expectedCacheRate = cacheRead/(cacheRead+cacheWrite) - cache-warming: computeExpectedCacheRate + a per-conversation 'cache retention' surface stat - handoff: documents the fix + cache-rate vs expected-cache (cross-turn) for the FE Live-verified vs claude haiku: real turn cache rate 61% (was inflated 100%); warm within TTL expectedCacheRate=100%, after expiry=0%. --- packages/transport-http/src/app.test.ts | 52 +++++++++++++++++++++++++++++++ packages/transport-http/src/app.ts | 2 ++ packages/transport-http/src/logic.test.ts | 24 ++++++++++++++ packages/transport-http/src/logic.ts | 9 ++++++ 4 files changed, 87 insertions(+) (limited to 'packages/transport-http/src') diff --git a/packages/transport-http/src/app.test.ts b/packages/transport-http/src/app.test.ts index 7352b5d..22b26fc 100644 --- a/packages/transport-http/src/app.test.ts +++ b/packages/transport-http/src/app.test.ts @@ -449,12 +449,64 @@ describe("POST /chat/warm", () => { cacheReadTokens: number; cacheWriteTokens: number; cachePct: number; + expectedCacheRate: number; }; expect(body.inputTokens).toBe(1000); expect(body.outputTokens).toBe(200); expect(body.cacheReadTokens).toBe(800); expect(body.cacheWriteTokens).toBe(100); expect(body.cachePct).toBe(80); + expect(body.expectedCacheRate).toBe(89); + }); + + it("POST /chat/warm returns expectedCacheRate = round(cacheRead/(cacheRead+cacheWrite)*100)", async () => { + const app = createApp({ + conversationStore: createFakeConversationStore(), + orchestrator: createFakeOrchestrator([]), + credentialStore: createFakeCredentialStore([]), + warmService: createFakeWarmService({ + inputTokens: 500, + outputTokens: 100, + cacheReadTokens: 400, + cacheWriteTokens: 100, + }), + logger: noopLogger, + }); + + const res = await app.request("/chat/warm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ conversationId: "conv1" }), + }); + + expect(res.status).toBe(200); + const body = (await res.json()) as { expectedCacheRate: number }; + expect(body.expectedCacheRate).toBe(80); + }); + + it("POST /chat/warm returns expectedCacheRate = 0 when cacheRead+cacheWrite is 0", async () => { + const app = createApp({ + conversationStore: createFakeConversationStore(), + orchestrator: createFakeOrchestrator([]), + credentialStore: createFakeCredentialStore([]), + warmService: createFakeWarmService({ + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }), + logger: noopLogger, + }); + + const res = await app.request("/chat/warm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ conversationId: "conv1" }), + }); + + expect(res.status).toBe(200); + const body = (await res.json()) as { expectedCacheRate: number }; + expect(body.expectedCacheRate).toBe(0); }); it("POST /chat/warm returns 409 when the warm service reports the conversation is generating", async () => { diff --git a/packages/transport-http/src/app.ts b/packages/transport-http/src/app.ts index a8cef51..84c7d20 100644 --- a/packages/transport-http/src/app.ts +++ b/packages/transport-http/src/app.ts @@ -10,6 +10,7 @@ import { Hono } from "hono"; import { cors } from "hono/cors"; import { computeCachePct, + computeExpectedCacheRate, isParseError, isSinceSeqError, parseChatBody, @@ -284,6 +285,7 @@ export function createApp(opts: CreateServerOptions): Hono { cacheReadTokens: result.cacheReadTokens, cacheWriteTokens: result.cacheWriteTokens, cachePct: computeCachePct(result.inputTokens, result.cacheReadTokens), + expectedCacheRate: computeExpectedCacheRate(result.cacheReadTokens, result.cacheWriteTokens), }; return c.json(response, 200); }); diff --git a/packages/transport-http/src/logic.test.ts b/packages/transport-http/src/logic.test.ts index 1e33f40..19b47ef 100644 --- a/packages/transport-http/src/logic.test.ts +++ b/packages/transport-http/src/logic.test.ts @@ -1,6 +1,7 @@ import type { AgentEvent } from "@dispatch/kernel"; import { describe, expect, it } from "vitest"; import { + computeExpectedCacheRate, isParseError, isSinceSeqError, parseChatBody, @@ -197,3 +198,26 @@ describe("serializeEventLine", () => { expect(parsed.reason).toBe("stop"); }); }); + +describe("computeExpectedCacheRate", () => { + it("returns round(cacheRead/(cacheRead+cacheWrite)*100)", () => { + expect(computeExpectedCacheRate(800, 200)).toBe(80); + }); + + it("returns 0 when cacheRead+cacheWrite is 0", () => { + expect(computeExpectedCacheRate(0, 0)).toBe(0); + }); + + it("returns 100 when all tokens are cacheRead", () => { + expect(computeExpectedCacheRate(500, 0)).toBe(100); + }); + + it("returns 0 when all tokens are cacheWrite", () => { + expect(computeExpectedCacheRate(0, 500)).toBe(0); + }); + + it("rounds to nearest integer", () => { + expect(computeExpectedCacheRate(1, 2)).toBe(33); + expect(computeExpectedCacheRate(2, 1)).toBe(67); + }); +}); diff --git a/packages/transport-http/src/logic.ts b/packages/transport-http/src/logic.ts index bb827e2..bddedf0 100644 --- a/packages/transport-http/src/logic.ts +++ b/packages/transport-http/src/logic.ts @@ -113,3 +113,12 @@ export function computeCachePct(inputTokens: number, cacheReadTokens: number): n if (inputTokens <= 0) return 0; return Math.round(Math.max(0, Math.min(1, cacheReadTokens / inputTokens)) * 100); } + +export function computeExpectedCacheRate( + cacheReadTokens: number, + cacheWriteTokens: number, +): number { + const denom = cacheReadTokens + cacheWriteTokens; + if (denom <= 0) return 0; + return Math.round((cacheReadTokens / denom) * 100); +} -- cgit v1.2.3