diff options
| author | Adam Malczewski <[email protected]> | 2026-06-10 10:16:20 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-10 10:16:20 +0900 |
| commit | 80e14ab59732aabbf06035d13138500f133e921d (patch) | |
| tree | be3c1c118ceeb207d7ff218aa5d7ace06e683710 /packages/throughput-store/src | |
| parent | 5ff460688519e48fd0bfab893ebaed4258dee789 (diff) | |
| download | dispatch-80e14ab59732aabbf06035d13138500f133e921d.tar.gz dispatch-80e14ab59732aabbf06035d13138500f133e921d.zip | |
feat: per-model throughput (tok/s) tracking + metrics endpoint
New throughput-store extension records one token-weighted sample per turn
(model, output tokens, pure generation time = Σ step genTotalMs) into a
day-bucketed KV store, and aggregates per-model tok/s = Σtokens / Σgen-seconds
over a day/week/month (server-local boundaries; week = ISO Mon–Sun).
transport-http records a sample per turn (logged) and serves
GET /metrics/throughput?period=day|week|month&date=<...>. The response is typed
as transport-contract's ThroughputResponse, so store/wire drift is a compile
error. Pure period + aggregate logic fully unit-tested.
Diffstat (limited to 'packages/throughput-store/src')
| -rw-r--r-- | packages/throughput-store/src/aggregate.test.ts | 50 | ||||
| -rw-r--r-- | packages/throughput-store/src/aggregate.ts | 72 | ||||
| -rw-r--r-- | packages/throughput-store/src/extension.ts | 24 | ||||
| -rw-r--r-- | packages/throughput-store/src/index.ts | 16 | ||||
| -rw-r--r-- | packages/throughput-store/src/period.test.ts | 67 | ||||
| -rw-r--r-- | packages/throughput-store/src/period.ts | 113 | ||||
| -rw-r--r-- | packages/throughput-store/src/service.ts | 5 | ||||
| -rw-r--r-- | packages/throughput-store/src/store.test.ts | 72 | ||||
| -rw-r--r-- | packages/throughput-store/src/store.ts | 83 |
9 files changed, 502 insertions, 0 deletions
diff --git a/packages/throughput-store/src/aggregate.test.ts b/packages/throughput-store/src/aggregate.test.ts new file mode 100644 index 0000000..27b800c --- /dev/null +++ b/packages/throughput-store/src/aggregate.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { aggregateSamples, type ThroughputSample } from "./aggregate.js"; + +const S = (model: string, ts: number, outputTokens: number, genMs: number): ThroughputSample => ({ + model, + ts, + outputTokens, + genMs, +}); + +describe("aggregateSamples", () => { + it("token-weights tok/s (Σtokens / Σgen-seconds), so large turns dominate", () => { + const samples = [ + S("claude/haiku", 100, 1000, 10_000), // 100 tok/s, big turn + S("claude/haiku", 200, 10, 1000), // 10 tok/s, small turn + ]; + const [row] = aggregateSamples(samples, 0, 1000); + expect(row?.model).toBe("claude/haiku"); + // 1010 tokens / 11 s = 91.82, NOT the simple mean (55) + expect(row?.tokensPerSecond).toBeCloseTo(91.82, 1); + expect(row?.totalOutputTokens).toBe(1010); + expect(row?.totalGenMs).toBe(11_000); + expect(row?.turns).toBe(2); + }); + + it("excludes samples outside the [start, end) range", () => { + const samples = [S("m", 50, 100, 1000), S("m", 500, 100, 1000), S("m", 1500, 999, 1000)]; + const [row] = aggregateSamples(samples, 100, 1000); + expect(row?.turns).toBe(1); // only ts=500 is in [100, 1000) + expect(row?.totalOutputTokens).toBe(100); + }); + + it("groups by model and sorts by tok/s descending", () => { + const samples = [ + S("slow", 10, 50, 5000), // 10 tok/s + S("fast", 10, 500, 1000), // 500 tok/s + ]; + const rows = aggregateSamples(samples, 0, 100); + expect(rows.map((r) => r.model)).toEqual(["fast", "slow"]); + }); + + it("reports 0 tok/s when generation time is 0 (avoids divide-by-zero)", () => { + const [row] = aggregateSamples([S("m", 10, 100, 0)], 0, 100); + expect(row?.tokensPerSecond).toBe(0); + }); + + it("returns an empty list when no samples match", () => { + expect(aggregateSamples([], 0, 100)).toEqual([]); + }); +}); diff --git a/packages/throughput-store/src/aggregate.ts b/packages/throughput-store/src/aggregate.ts new file mode 100644 index 0000000..2437d9f --- /dev/null +++ b/packages/throughput-store/src/aggregate.ts @@ -0,0 +1,72 @@ +/** + * Pure throughput aggregation. + * + * Per model, tokens-per-second is the TOKEN-WEIGHTED average: + * + * tok/s = Σ(output_tokens) / Σ(generation_seconds) + * + * i.e. total tokens over total generation time across the period's turns. This + * makes a turn that generated more tokens count proportionally more than a small + * turn — large turns dominate, exactly as intended. Generation time is the pure + * decode time (excludes tool-execution waits). + */ + +export interface ThroughputSample { + readonly model: string; + /** Epoch-ms the turn completed. */ + readonly ts: number; + /** Output tokens generated in the turn. */ + readonly outputTokens: number; + /** Pure generation time for the turn (ms), summed across its steps. */ + readonly genMs: number; +} + +export interface ModelThroughput { + readonly model: string; + /** Token-weighted average tokens/second over the period. */ + readonly tokensPerSecond: number; + readonly totalOutputTokens: number; + readonly totalGenMs: number; + /** Number of turns that contributed. */ + readonly turns: number; +} + +/** + * Aggregate samples within the half-open range `[start, end)` into per-model + * throughput, sorted by tok/s descending (ties broken by model name). + */ +export function aggregateSamples( + samples: readonly ThroughputSample[], + start: number, + end: number, +): ModelThroughput[] { + const byModel = new Map<string, { tokens: number; genMs: number; turns: number }>(); + + for (const s of samples) { + if (s.ts < start || s.ts >= end) continue; + const acc = byModel.get(s.model) ?? { tokens: 0, genMs: 0, turns: 0 }; + acc.tokens += s.outputTokens; + acc.genMs += s.genMs; + acc.turns += 1; + byModel.set(s.model, acc); + } + + const result: ModelThroughput[] = []; + for (const [model, acc] of byModel) { + const tokensPerSecond = acc.genMs > 0 ? round2(acc.tokens / (acc.genMs / 1000)) : 0; + result.push({ + model, + tokensPerSecond, + totalOutputTokens: acc.tokens, + totalGenMs: acc.genMs, + turns: acc.turns, + }); + } + + result.sort((a, b) => b.tokensPerSecond - a.tokensPerSecond || a.model.localeCompare(b.model)); + return result; +} + +function round2(n: number): number { + return Math.round(n * 100) / 100; +} diff --git a/packages/throughput-store/src/extension.ts b/packages/throughput-store/src/extension.ts new file mode 100644 index 0000000..01d1549 --- /dev/null +++ b/packages/throughput-store/src/extension.ts @@ -0,0 +1,24 @@ +import type { Extension, HostAPI, Manifest } from "@dispatch/kernel"; +import { throughputStoreHandle } from "./service.js"; +import { createThroughputStore } from "./store.js"; + +export const manifest: Manifest = { + id: "throughput-store", + name: "Throughput Store", + version: "0.0.0", + apiVersion: "^0.1.0", + trust: "bundled", + capabilities: { db: true }, + contributes: { services: ["throughput-store/store"] }, + activation: "eager", +}; + +export const extension: Extension = { + manifest, + activate: (host: HostAPI) => { + const storage = host.storage("throughput-store"); + const store = createThroughputStore({ storage, logger: host.logger }); + host.provideService(throughputStoreHandle, store); + host.logger.info("throughput-store: registered"); + }, +}; diff --git a/packages/throughput-store/src/index.ts b/packages/throughput-store/src/index.ts new file mode 100644 index 0000000..24ebaba --- /dev/null +++ b/packages/throughput-store/src/index.ts @@ -0,0 +1,16 @@ +export { + aggregateSamples, + type ModelThroughput, + type ThroughputSample, +} from "./aggregate.js"; +export { extension, manifest } from "./extension.js"; +export { dayKeyOf, type Period, resolvePeriod } from "./period.js"; +export { throughputStoreHandle } from "./service.js"; +export { + createThroughputStore, + type ThroughputQuery, + ThroughputQueryError, + type ThroughputReport, + type ThroughputStore, + type ThroughputStoreDeps, +} from "./store.js"; diff --git a/packages/throughput-store/src/period.test.ts b/packages/throughput-store/src/period.test.ts new file mode 100644 index 0000000..b39437c --- /dev/null +++ b/packages/throughput-store/src/period.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { dayKeyOf, resolvePeriod } from "./period.js"; + +describe("dayKeyOf", () => { + it("formats a local YYYY-MM-DD key", () => { + // Build a local-midnight timestamp so the key is timezone-stable. + const ts = new Date(2026, 5, 10).getTime(); + expect(dayKeyOf(ts)).toBe("2026-06-10"); + }); +}); + +describe("resolvePeriod day", () => { + it("spans a single local day", () => { + const r = resolvePeriod("day", "2026-06-10"); + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.dayKeys).toEqual(["2026-06-10"]); + expect(r.start).toBe(new Date(2026, 5, 10).getTime()); + expect(r.end).toBe(new Date(2026, 5, 11).getTime()); + }); + + it("rejects malformed / impossible dates", () => { + expect(resolvePeriod("day", "2026-13-01").ok).toBe(false); + expect(resolvePeriod("day", "2026-02-30").ok).toBe(false); + expect(resolvePeriod("day", "nope").ok).toBe(false); + expect(resolvePeriod("day", "2026-06").ok).toBe(false); + }); +}); + +describe("resolvePeriod week", () => { + it("spans the Monday–Sunday ISO week containing the date", () => { + const r = resolvePeriod("week", "2026-06-10"); + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.dayKeys).toHaveLength(7); + // start is a Monday (local) + expect(new Date(r.start).getDay()).toBe(1); + // the queried date falls within the week + expect(r.dayKeys).toContain("2026-06-10"); + // end is exactly 7 local days after start + expect(new Date(r.end).getDay()).toBe(1); + }); +}); + +describe("resolvePeriod month", () => { + it("spans a full calendar month", () => { + const r = resolvePeriod("month", "2026-06"); + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.dayKeys).toHaveLength(30); // June has 30 days + expect(r.dayKeys[0]).toBe("2026-06-01"); + expect(r.dayKeys[29]).toBe("2026-06-30"); + expect(r.start).toBe(new Date(2026, 5, 1).getTime()); + expect(r.end).toBe(new Date(2026, 6, 1).getTime()); + }); + + it("handles February length", () => { + const r = resolvePeriod("month", "2026-02"); + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.dayKeys).toHaveLength(28); + }); + + it("rejects a YYYY-MM-DD date for month", () => { + expect(resolvePeriod("month", "2026-06-10").ok).toBe(false); + }); +}); diff --git a/packages/throughput-store/src/period.ts b/packages/throughput-store/src/period.ts new file mode 100644 index 0000000..d8225f8 --- /dev/null +++ b/packages/throughput-store/src/period.ts @@ -0,0 +1,113 @@ +/** + * Pure period math for throughput aggregation. + * + * All boundaries are computed in the SERVER'S LOCAL timezone (a single-operator + * self-hosted assumption): + * - day: `YYYY-MM-DD` → [local midnight, next local midnight) + * - week: `YYYY-MM-DD` → the ISO week (Mon–Sun) CONTAINING that date + * - month: `YYYY-MM` → [first of month, first of next month) + * + * A resolved period yields the half-open `[start, end)` epoch-ms range plus the + * list of local `YYYY-MM-DD` day keys it spans (used to address the per-day + * sample buckets in storage). + */ + +export type Period = "day" | "week" | "month"; + +export interface ResolvedPeriod { + readonly ok: true; + /** Inclusive start, epoch-ms (local midnight). */ + readonly start: number; + /** Exclusive end, epoch-ms (local midnight). */ + readonly end: number; + /** Local `YYYY-MM-DD` day keys this period spans, in order. */ + readonly dayKeys: readonly string[]; + /** The normalized input date string. */ + readonly date: string; +} + +export interface PeriodError { + readonly ok: false; + readonly error: string; +} + +function pad2(n: number): string { + return n < 10 ? `0${n}` : String(n); +} + +/** Local `YYYY-MM-DD` key for an epoch-ms timestamp. */ +export function dayKeyOf(ts: number): string { + const d = new Date(ts); + return `${d.getFullYear()}-${pad2(d.getMonth() + 1)}-${pad2(d.getDate())}`; +} + +/** Local `YYYY-MM-DD` key for a local calendar date. */ +function dayKey(year: number, monthIndex: number, day: number): string { + const d = new Date(year, monthIndex, day); + return `${d.getFullYear()}-${pad2(d.getMonth() + 1)}-${pad2(d.getDate())}`; +} + +function parseYmd(date: string): { y: number; m: number; d: number } | null { + if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return null; + const [y, m, d] = date.split("-").map(Number) as [number, number, number]; + if (m < 1 || m > 12 || d < 1 || d > 31) return null; + // Reject impossible dates (e.g. 2026-02-30) by round-tripping through Date. + const probe = new Date(y, m - 1, d); + if (probe.getFullYear() !== y || probe.getMonth() !== m - 1 || probe.getDate() !== d) { + return null; + } + return { y, m, d }; +} + +function parseYm(date: string): { y: number; m: number } | null { + if (!/^\d{4}-\d{2}$/.test(date)) return null; + const [y, m] = date.split("-").map(Number) as [number, number]; + if (m < 1 || m > 12) return null; + return { y, m }; +} + +/** + * Resolve a `(period, date)` pair into a concrete `[start, end)` range + the day + * keys it covers. Returns a `PeriodError` for malformed input. + */ +export function resolvePeriod(period: Period, date: string): ResolvedPeriod | PeriodError { + if (period === "day") { + const p = parseYmd(date); + if (!p) return { ok: false, error: `invalid day date "${date}" (expected YYYY-MM-DD)` }; + const start = new Date(p.y, p.m - 1, p.d).getTime(); + const end = new Date(p.y, p.m - 1, p.d + 1).getTime(); + return { ok: true, start, end, dayKeys: [dayKey(p.y, p.m - 1, p.d)], date }; + } + + if (period === "week") { + const p = parseYmd(date); + if (!p) return { ok: false, error: `invalid week date "${date}" (expected YYYY-MM-DD)` }; + // ISO week: Monday-based. JS getDay() is 0=Sun..6=Sat. + const base = new Date(p.y, p.m - 1, p.d); + const offset = (base.getDay() + 6) % 7; // days since Monday + const monStart = new Date(p.y, p.m - 1, p.d - offset); + const start = monStart.getTime(); + const end = new Date( + monStart.getFullYear(), + monStart.getMonth(), + monStart.getDate() + 7, + ).getTime(); + const dayKeys: string[] = []; + for (let i = 0; i < 7; i++) { + dayKeys.push(dayKey(monStart.getFullYear(), monStart.getMonth(), monStart.getDate() + i)); + } + return { ok: true, start, end, dayKeys, date }; + } + + // month + const p = parseYm(date); + if (!p) return { ok: false, error: `invalid month date "${date}" (expected YYYY-MM)` }; + const start = new Date(p.y, p.m - 1, 1).getTime(); + const end = new Date(p.y, p.m, 1).getTime(); + const lastDay = new Date(p.y, p.m, 0).getDate(); + const dayKeys: string[] = []; + for (let d = 1; d <= lastDay; d++) { + dayKeys.push(dayKey(p.y, p.m - 1, d)); + } + return { ok: true, start, end, dayKeys, date }; +} diff --git a/packages/throughput-store/src/service.ts b/packages/throughput-store/src/service.ts new file mode 100644 index 0000000..d3a2184 --- /dev/null +++ b/packages/throughput-store/src/service.ts @@ -0,0 +1,5 @@ +import { defineService } from "@dispatch/kernel"; +import type { ThroughputStore } from "./store.js"; + +/** Typed service handle for the throughput store (record + aggregate). */ +export const throughputStoreHandle = defineService<ThroughputStore>("throughput-store/store"); diff --git a/packages/throughput-store/src/store.test.ts b/packages/throughput-store/src/store.test.ts new file mode 100644 index 0000000..e81201a --- /dev/null +++ b/packages/throughput-store/src/store.test.ts @@ -0,0 +1,72 @@ +import type { StorageNamespace } from "@dispatch/kernel"; +import { describe, expect, it } from "vitest"; +import { dayKeyOf } from "./period.js"; +import { createThroughputStore, ThroughputQueryError } from "./store.js"; + +function memStorage(): StorageNamespace { + const map = new Map<string, string>(); + return { + get: async (k) => map.get(k) ?? null, + set: async (k, v) => { + map.set(k, v); + }, + delete: async (k) => { + map.delete(k); + }, + has: async (k) => map.has(k), + keys: async (prefix) => + [...map.keys()].filter((k) => (prefix === undefined ? true : k.startsWith(prefix))), + }; +} + +let id = 0; +const store = () => createThroughputStore({ storage: memStorage(), newId: () => `id${id++}` }); + +describe("ThroughputStore", () => { + it("records a sample and aggregates it for that day", async () => { + const s = store(); + const ts = new Date(2026, 5, 10, 12, 0, 0).getTime(); + await s.record({ model: "claude/haiku", ts, outputTokens: 300, genMs: 1500 }); + + const report = await s.aggregate({ period: "day", date: dayKeyOf(ts) }); + expect(report.models).toHaveLength(1); + expect(report.models[0]).toMatchObject({ + model: "claude/haiku", + totalOutputTokens: 300, + totalGenMs: 1500, + turns: 1, + tokensPerSecond: 200, // 300 / 1.5s + }); + }); + + it("does not lose concurrent samples (single-set writes)", async () => { + const s = store(); + const ts = new Date(2026, 5, 10, 9, 0, 0).getTime(); + await Promise.all([ + s.record({ model: "m", ts, outputTokens: 100, genMs: 1000 }), + s.record({ model: "m", ts, outputTokens: 100, genMs: 1000 }), + s.record({ model: "m", ts, outputTokens: 100, genMs: 1000 }), + ]); + const report = await s.aggregate({ period: "day", date: dayKeyOf(ts) }); + expect(report.models[0]?.turns).toBe(3); + expect(report.models[0]?.totalOutputTokens).toBe(300); + }); + + it("aggregates multiple days within a week", async () => { + const s = store(); + const mon = new Date(2026, 5, 8, 10, 0, 0).getTime(); // Mon 2026-06-08 + const wed = new Date(2026, 5, 10, 10, 0, 0).getTime(); + await s.record({ model: "m", ts: mon, outputTokens: 100, genMs: 1000 }); + await s.record({ model: "m", ts: wed, outputTokens: 200, genMs: 1000 }); + + const report = await s.aggregate({ period: "week", date: dayKeyOf(wed) }); + expect(report.models[0]?.turns).toBe(2); + expect(report.models[0]?.totalOutputTokens).toBe(300); + }); + + it("throws ThroughputQueryError on a malformed date", async () => { + await expect(store().aggregate({ period: "day", date: "garbage" })).rejects.toBeInstanceOf( + ThroughputQueryError, + ); + }); +}); diff --git a/packages/throughput-store/src/store.ts b/packages/throughput-store/src/store.ts new file mode 100644 index 0000000..94675b1 --- /dev/null +++ b/packages/throughput-store/src/store.ts @@ -0,0 +1,83 @@ +import type { Logger, StorageNamespace } from "@dispatch/kernel"; +import { aggregateSamples, type ModelThroughput, type ThroughputSample } from "./aggregate.js"; +import { dayKeyOf, type Period, resolvePeriod } from "./period.js"; + +export interface ThroughputReport { + readonly period: Period; + readonly date: string; + /** Inclusive start, epoch-ms. */ + readonly start: number; + /** Exclusive end, epoch-ms. */ + readonly end: number; + readonly models: readonly ModelThroughput[]; +} + +export interface ThroughputQuery { + readonly period: Period; + readonly date: string; +} + +/** + * Cross-conversation, per-model throughput store. Records one sample per + * completed turn and aggregates token-weighted tok/s over a day/week/month. + * Persistence is the injected KV namespace; samples are bucketed by local day + * key so a period query addresses only its own day buckets. + */ +export interface ThroughputStore { + readonly record: (sample: ThroughputSample) => Promise<void>; + readonly aggregate: (query: ThroughputQuery) => Promise<ThroughputReport>; +} + +export interface ThroughputStoreDeps { + readonly storage: StorageNamespace; + readonly logger?: Logger; + /** Injectable unique-id generator (default crypto.randomUUID). */ + readonly newId?: () => string; +} + +/** Thrown when a query's `(period, date)` is malformed. */ +export class ThroughputQueryError extends Error {} + +const SAMPLE_PREFIX = "sample"; + +export function createThroughputStore(deps: ThroughputStoreDeps): ThroughputStore { + const newId = deps.newId ?? (() => crypto.randomUUID()); + + return { + async record(sample) { + // Per-sample key under the local-day bucket → write is a single set + // (no read-modify-write, so concurrent turns can't lose a sample). + const day = dayKeyOf(sample.ts); + const key = `${SAMPLE_PREFIX}:${day}:${sample.ts}:${newId()}`; + await deps.storage.set(key, JSON.stringify(sample)); + }, + + async aggregate(query) { + const resolved = resolvePeriod(query.period, query.date); + if (!resolved.ok) throw new ThroughputQueryError(resolved.error); + + const samples: ThroughputSample[] = []; + for (const day of resolved.dayKeys) { + const keys = await deps.storage.keys(`${SAMPLE_PREFIX}:${day}:`); + for (const k of keys) { + const raw = await deps.storage.get(k); + if (raw === null) continue; + try { + samples.push(JSON.parse(raw) as ThroughputSample); + } catch { + // Skip a malformed row rather than failing the whole query. + } + } + } + + const models = aggregateSamples(samples, resolved.start, resolved.end); + return { + period: query.period, + date: resolved.date, + start: resolved.start, + end: resolved.end, + models, + }; + }, + }; +} |
