summaryrefslogtreecommitdiffhomepage
path: root/packages/transport-http/src
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-11 14:11:13 +0900
committerAdam Malczewski <[email protected]>2026-06-11 14:11:13 +0900
commit7ffb6b28f5b6bdbfc53ebed94fc68af557612189 (patch)
treee66d9ea9d326ef771cc473d81ca5716ff78b08a8 /packages/transport-http/src
parent763e5fb1c7fbfb4c7bbd43ffb935e42e5f5b5a42 (diff)
downloaddispatch-7ffb6b28f5b6bdbfc53ebed94fc68af557612189.tar.gz
dispatch-7ffb6b28f5b6bdbfc53ebed94fc68af557612189.zip
fix(cache-warming): accurate cache rate + expectedCacheRate (retention) metric
The Claude cache % read 100% whenever anything was cached, because the metric's denominator (inputTokens) excluded cached tokens on Anthropic. Fixed upstream in ../claude/provider-anthropic (inputTokens = total prompt); this commit adds the companion retention metric and exposes it: - transport-contract: WarmResponse += expectedCacheRate - transport-http: POST /chat/warm returns expectedCacheRate = cacheRead/(cacheRead+cacheWrite) - cache-warming: computeExpectedCacheRate + a per-conversation 'cache retention' surface stat - handoff: documents the fix + cache-rate vs expected-cache (cross-turn) for the FE Live-verified vs claude haiku: real turn cache rate 61% (was inflated 100%); warm within TTL expectedCacheRate=100%, after expiry=0%.
Diffstat (limited to 'packages/transport-http/src')
-rw-r--r--packages/transport-http/src/app.test.ts52
-rw-r--r--packages/transport-http/src/app.ts2
-rw-r--r--packages/transport-http/src/logic.test.ts24
-rw-r--r--packages/transport-http/src/logic.ts9
4 files changed, 87 insertions, 0 deletions
diff --git a/packages/transport-http/src/app.test.ts b/packages/transport-http/src/app.test.ts
index 7352b5d..22b26fc 100644
--- a/packages/transport-http/src/app.test.ts
+++ b/packages/transport-http/src/app.test.ts
@@ -449,12 +449,64 @@ describe("POST /chat/warm", () => {
cacheReadTokens: number;
cacheWriteTokens: number;
cachePct: number;
+ expectedCacheRate: number;
};
expect(body.inputTokens).toBe(1000);
expect(body.outputTokens).toBe(200);
expect(body.cacheReadTokens).toBe(800);
expect(body.cacheWriteTokens).toBe(100);
expect(body.cachePct).toBe(80);
+ expect(body.expectedCacheRate).toBe(89);
+ });
+
+ it("POST /chat/warm returns expectedCacheRate = round(cacheRead/(cacheRead+cacheWrite)*100)", async () => {
+ const app = createApp({
+ conversationStore: createFakeConversationStore(),
+ orchestrator: createFakeOrchestrator([]),
+ credentialStore: createFakeCredentialStore([]),
+ warmService: createFakeWarmService({
+ inputTokens: 500,
+ outputTokens: 100,
+ cacheReadTokens: 400,
+ cacheWriteTokens: 100,
+ }),
+ logger: noopLogger,
+ });
+
+ const res = await app.request("/chat/warm", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ conversationId: "conv1" }),
+ });
+
+ expect(res.status).toBe(200);
+ const body = (await res.json()) as { expectedCacheRate: number };
+ expect(body.expectedCacheRate).toBe(80);
+ });
+
+ it("POST /chat/warm returns expectedCacheRate = 0 when cacheRead+cacheWrite is 0", async () => {
+ const app = createApp({
+ conversationStore: createFakeConversationStore(),
+ orchestrator: createFakeOrchestrator([]),
+ credentialStore: createFakeCredentialStore([]),
+ warmService: createFakeWarmService({
+ inputTokens: 100,
+ outputTokens: 50,
+ cacheReadTokens: 0,
+ cacheWriteTokens: 0,
+ }),
+ logger: noopLogger,
+ });
+
+ const res = await app.request("/chat/warm", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ conversationId: "conv1" }),
+ });
+
+ expect(res.status).toBe(200);
+ const body = (await res.json()) as { expectedCacheRate: number };
+ expect(body.expectedCacheRate).toBe(0);
});
it("POST /chat/warm returns 409 when the warm service reports the conversation is generating", async () => {
diff --git a/packages/transport-http/src/app.ts b/packages/transport-http/src/app.ts
index a8cef51..84c7d20 100644
--- a/packages/transport-http/src/app.ts
+++ b/packages/transport-http/src/app.ts
@@ -10,6 +10,7 @@ import { Hono } from "hono";
import { cors } from "hono/cors";
import {
computeCachePct,
+ computeExpectedCacheRate,
isParseError,
isSinceSeqError,
parseChatBody,
@@ -284,6 +285,7 @@ export function createApp(opts: CreateServerOptions): Hono {
cacheReadTokens: result.cacheReadTokens,
cacheWriteTokens: result.cacheWriteTokens,
cachePct: computeCachePct(result.inputTokens, result.cacheReadTokens),
+ expectedCacheRate: computeExpectedCacheRate(result.cacheReadTokens, result.cacheWriteTokens),
};
return c.json(response, 200);
});
diff --git a/packages/transport-http/src/logic.test.ts b/packages/transport-http/src/logic.test.ts
index 1e33f40..19b47ef 100644
--- a/packages/transport-http/src/logic.test.ts
+++ b/packages/transport-http/src/logic.test.ts
@@ -1,6 +1,7 @@
import type { AgentEvent } from "@dispatch/kernel";
import { describe, expect, it } from "vitest";
import {
+ computeExpectedCacheRate,
isParseError,
isSinceSeqError,
parseChatBody,
@@ -197,3 +198,26 @@ describe("serializeEventLine", () => {
expect(parsed.reason).toBe("stop");
});
});
+
+describe("computeExpectedCacheRate", () => {
+ it("returns round(cacheRead/(cacheRead+cacheWrite)*100)", () => {
+ expect(computeExpectedCacheRate(800, 200)).toBe(80);
+ });
+
+ it("returns 0 when cacheRead+cacheWrite is 0", () => {
+ expect(computeExpectedCacheRate(0, 0)).toBe(0);
+ });
+
+ it("returns 100 when all tokens are cacheRead", () => {
+ expect(computeExpectedCacheRate(500, 0)).toBe(100);
+ });
+
+ it("returns 0 when all tokens are cacheWrite", () => {
+ expect(computeExpectedCacheRate(0, 500)).toBe(0);
+ });
+
+ it("rounds to nearest integer", () => {
+ expect(computeExpectedCacheRate(1, 2)).toBe(33);
+ expect(computeExpectedCacheRate(2, 1)).toBe(67);
+ });
+});
diff --git a/packages/transport-http/src/logic.ts b/packages/transport-http/src/logic.ts
index bb827e2..bddedf0 100644
--- a/packages/transport-http/src/logic.ts
+++ b/packages/transport-http/src/logic.ts
@@ -113,3 +113,12 @@ export function computeCachePct(inputTokens: number, cacheReadTokens: number): n
if (inputTokens <= 0) return 0;
return Math.round(Math.max(0, Math.min(1, cacheReadTokens / inputTokens)) * 100);
}
+
+export function computeExpectedCacheRate(
+ cacheReadTokens: number,
+ cacheWriteTokens: number,
+): number {
+ const denom = cacheReadTokens + cacheWriteTokens;
+ if (denom <= 0) return 0;
+ return Math.round((cacheReadTokens / denom) * 100);
+}