diff options
| author | Adam Malczewski <[email protected]> | 2026-05-30 18:53:42 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-05-30 18:53:42 +0900 |
| commit | 2228691e14be2368394e38e600bfa2ce227487b1 (patch) | |
| tree | d6b3cfffd11dc9f7eec8c88f5fc97a7ec81e61a0 /packages/frontend/src/lib | |
| parent | 497b397e873f96d6fde3d8a44b3318e1ee1cbef4 (diff) | |
| download | dispatch-2228691e14be2368394e38e600bfa2ce227487b1.tar.gz dispatch-2228691e14be2368394e38e600bfa2ce227487b1.zip | |
feat(cache): Anthropic prompt caching, usage telemetry, and Cache Rate view
- send prompt-caching + oauth anthropic-beta headers on the Claude OAuth provider
- restructure the OAuth request body (billing header, identity split, relocate
third-party system prompt to the first user message) to match Claude Code
- apply rolling cache_control breakpoints and group a turn's tool results into a
single role:tool message for correct breakpoint placement
- emit per-step usage events (cache read/write split) and add the Cache Rate
sidebar panel
- dedup byte-identical tool calls within a single batch
Diffstat (limited to 'packages/frontend/src/lib')
| -rw-r--r-- | packages/frontend/src/lib/components/CacheRatePanel.svelte | 129 | ||||
| -rw-r--r-- | packages/frontend/src/lib/components/SidebarPanel.svelte | 14 | ||||
| -rw-r--r-- | packages/frontend/src/lib/tabs.svelte.ts | 30 | ||||
| -rw-r--r-- | packages/frontend/src/lib/types.ts | 32 |
4 files changed, 202 insertions, 3 deletions
diff --git a/packages/frontend/src/lib/components/CacheRatePanel.svelte b/packages/frontend/src/lib/components/CacheRatePanel.svelte new file mode 100644 index 0000000..c35cbb5 --- /dev/null +++ b/packages/frontend/src/lib/components/CacheRatePanel.svelte @@ -0,0 +1,129 @@ +<script lang="ts"> +import type { CacheStats } from "../types.js"; + +const { + cacheStats = null, + tabTitle = null, +}: { + cacheStats?: CacheStats | null; + tabTitle?: string | null; +} = $props(); + +// Cache hit rate = cached-read tokens / total prompt tokens. `inputTokens` is +// the TOTAL prompt (fresh + cache read + cache write), so this is the share of +// the prompt that was served from Anthropic's prompt cache. +function rate(read: number, totalInput: number): number { + if (totalInput <= 0) return 0; + return Math.max(0, Math.min(1, read / totalInput)); +} + +// For caching, a HIGH hit rate is GOOD — invert the usual color thresholds. +function rateClass(r: number): string { + if (r >= 0.7) return "progress-success"; + if (r >= 0.3) return "progress-warning"; + return "progress-error"; +} + +function fmt(n: number): string { + return n.toLocaleString(); +} + +const hitRate = $derived(cacheStats ? rate(cacheStats.cacheReadTokens, cacheStats.inputTokens) : 0); +const hitPct = $derived(Math.round(hitRate * 100)); +const uncached = $derived( + cacheStats + ? Math.max(0, cacheStats.inputTokens - cacheStats.cacheReadTokens - cacheStats.cacheWriteTokens) + : 0, +); +const lastHitPct = $derived( + cacheStats?.last + ? Math.round(rate(cacheStats.last.cacheReadTokens, cacheStats.last.inputTokens) * 100) + : 0, +); +</script> + +<div class="flex flex-col gap-3 flex-1 min-h-0 overflow-y-auto"> + {#if !cacheStats || cacheStats.requests === 0} + <p class="text-xs text-base-content/50"> + No cache data yet. Send a message to a Claude model — prompt-cache usage + appears here after the first response. + </p> + {:else} + <div class="bg-base-200 rounded-lg p-2"> + <div class="flex items-center gap-1.5 mb-2"> + <span class="text-xs font-semibold">Cache Hit Rate</span> + {#if tabTitle} + <span class="badge badge-xs badge-ghost">{tabTitle}</span> + {/if} + <span class="badge badge-xs ml-auto">{cacheStats.requests} req</span> + </div> + + <!-- Headline cumulative hit rate --> + <div class="flex flex-col gap-0.5"> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50">Session (this tab)</span> + <span class="text-xs font-mono">{hitPct}%</span> + </div> + <progress + class="progress w-full h-2 {rateClass(hitRate)}" + value={hitPct} + max="100" + ></progress> + </div> + + <!-- Most recent request --> + {#if cacheStats.last} + <div class="flex flex-col gap-0.5 mt-2"> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50">Last request</span> + <span class="text-xs font-mono">{lastHitPct}%</span> + </div> + <progress + class="progress w-full h-2 {rateClass(lastHitPct / 100)}" + value={lastHitPct} + max="100" + ></progress> + </div> + {/if} + </div> + + <!-- Token breakdown (cumulative, this tab) --> + <div class="bg-base-200 rounded-lg p-2"> + <div class="text-xs font-semibold mb-1.5">Tokens (cumulative)</div> + <div class="flex flex-col gap-1 pl-1"> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50"> + <span class="badge badge-xs badge-success badge-soft mr-1">read</span>Cache hits + </span> + <span class="text-xs font-mono">{fmt(cacheStats.cacheReadTokens)}</span> + </div> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50"> + <span class="badge badge-xs badge-warning badge-soft mr-1">write</span>Cache writes + </span> + <span class="text-xs font-mono">{fmt(cacheStats.cacheWriteTokens)}</span> + </div> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50"> + <span class="badge badge-xs badge-error badge-soft mr-1">fresh</span>Uncached input + </span> + <span class="text-xs font-mono">{fmt(uncached)}</span> + </div> + <div class="border-t border-base-300 my-0.5"></div> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50">Total input</span> + <span class="text-xs font-mono">{fmt(cacheStats.inputTokens)}</span> + </div> + <div class="flex items-center justify-between"> + <span class="text-xs text-base-content/50">Output</span> + <span class="text-xs font-mono">{fmt(cacheStats.outputTokens)}</span> + </div> + </div> + </div> + + <p class="text-xs text-base-content/40"> + Cache reads cost ~10% of fresh input; writes cost ~25% more. A high hit + rate after the first turn means caching is working. Resets on reload. + </p> + {/if} +</div> diff --git a/packages/frontend/src/lib/components/SidebarPanel.svelte b/packages/frontend/src/lib/components/SidebarPanel.svelte index fca53b7..33b2158 100644 --- a/packages/frontend/src/lib/components/SidebarPanel.svelte +++ b/packages/frontend/src/lib/components/SidebarPanel.svelte @@ -1,6 +1,7 @@ <script lang="ts"> import { loadSidebarPanels, saveSidebarPanels } from "../sidebar-storage.js"; -import type { KeyInfo, LogEntry, TaskItem } from "../types.js"; +import type { CacheStats, KeyInfo, LogEntry, TaskItem } from "../types.js"; +import CacheRatePanel from "./CacheRatePanel.svelte"; import ClaudeReset from "./ClaudeReset.svelte"; import ConfigPanel from "./ConfigPanel.svelte"; import KeyUsage from "./KeyUsage.svelte"; @@ -23,6 +24,8 @@ interface AgentInfo { const { keys = [], tasks = [], + cacheStats = null, + cacheTabTitle = null, permissionLog = [], apiBase = "", activeKeyId = null, @@ -41,6 +44,8 @@ const { }: { keys?: KeyInfo[]; tasks?: TaskItem[]; + cacheStats?: CacheStats | null; + cacheTabTitle?: string | null; permissionLog?: LogEntry[]; apiBase?: string; activeKeyId?: string | null; @@ -82,6 +87,7 @@ const viewOptions = [ "Select a view", "Chat Settings", "Key Usage", + "Cache Rate", "Claude Reset", "Model Status", "Tasks", @@ -97,12 +103,12 @@ function addPanel() { function panelClass(selected: string): string { const base = "bg-base-200 rounded-lg p-3 flex flex-col min-h-0"; - const fill = selected === "Key Usage" || selected === "Tasks"; + const fill = selected === "Key Usage" || selected === "Tasks" || selected === "Cache Rate"; return fill ? `${base} flex-1` : base; } function contentClass(selected: string): string { - const fill = selected === "Key Usage" || selected === "Tasks"; + const fill = selected === "Key Usage" || selected === "Tasks" || selected === "Cache Rate"; return fill ? "mt-2 flex-1 min-h-0" : "mt-2"; } </script> @@ -156,6 +162,8 @@ function contentClass(selected: string): string { /> {:else if panel.selected === "Key Usage"} <KeyUsage {keys} {apiBase} /> + {:else if panel.selected === "Cache Rate"} + <CacheRatePanel {cacheStats} tabTitle={cacheTabTitle} /> {:else if panel.selected === "Claude Reset"} <ClaudeReset {apiBase} /> {:else if panel.selected === "Model Status"} diff --git a/packages/frontend/src/lib/tabs.svelte.ts b/packages/frontend/src/lib/tabs.svelte.ts index bb8f4cf..6e2d157 100644 --- a/packages/frontend/src/lib/tabs.svelte.ts +++ b/packages/frontend/src/lib/tabs.svelte.ts @@ -12,6 +12,7 @@ import { config } from "./config.js"; import { appSettings } from "./settings.svelte.js"; import type { AgentEvent, + CacheStats, ChatMessage, Chunk, DebugInfo, @@ -89,6 +90,12 @@ export interface Tab { oldestLoadedSeq: number | null; /** Total number of messages for this tab on the backend */ totalMessages: number; + /** + * Cumulative prompt-cache token telemetry for this tab since the page + * loaded (in-memory only — resets on reload). Undefined until the first + * `usage` event arrives. Drives the "Cache Rate" sidebar view. + */ + cacheStats?: CacheStats; } /** @@ -846,6 +853,29 @@ export function createTabStore() { applyChunkEvent(tabId, event); break; } + case "usage": { + if (!tabId) break; + const tab = getTabById(tabId); + if (!tab) break; + const u = event.usage; + const prev = tab.cacheStats; + updateTab(tabId, { + cacheStats: { + inputTokens: (prev?.inputTokens ?? 0) + u.inputTokens, + outputTokens: (prev?.outputTokens ?? 0) + u.outputTokens, + cacheReadTokens: (prev?.cacheReadTokens ?? 0) + u.cacheReadTokens, + cacheWriteTokens: (prev?.cacheWriteTokens ?? 0) + u.cacheWriteTokens, + requests: (prev?.requests ?? 0) + 1, + last: { + inputTokens: u.inputTokens, + outputTokens: u.outputTokens, + cacheReadTokens: u.cacheReadTokens, + cacheWriteTokens: u.cacheWriteTokens, + }, + }, + }); + break; + } case "done": { if (!tabId) break; const tab5 = getTabById(tabId); diff --git a/packages/frontend/src/lib/types.ts b/packages/frontend/src/lib/types.ts index f9add94..8c34d69 100644 --- a/packages/frontend/src/lib/types.ts +++ b/packages/frontend/src/lib/types.ts @@ -12,6 +12,29 @@ export interface DebugInfo { } /** + * Per-tab prompt-cache telemetry, accumulated from the `usage` AgentEvent + * (one per LLM round-trip). Token counts are cumulative across the session + * since the page loaded; `last` holds the most recent request's split. Powers + * the "Cache Rate" sidebar view. The cache hit rate is + * `cacheReadTokens / inputTokens` (inputTokens is the TOTAL prompt, including + * cached tokens). + */ +export interface CacheStats { + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + /** Number of LLM requests (usage events) counted. */ + requests: number; + last: { + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + } | null; +} + +/** * Mirror of the core `Chunk` union (see packages/core/src/types/index.ts). * * Wire-format symmetry MUST be kept with core. If you change one, change @@ -119,6 +142,15 @@ export type AgentEvent = type: "tool-result"; toolResult: { toolCallId: string; result: string; isError: boolean }; } + | { + type: "usage"; + usage: { + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + }; + } | { type: "error"; error: string } | { type: "notice"; message: string } | { type: "model-changed"; keyId: string; modelId: string } |
