diff options
| author | Adam Malczewski <[email protected]> | 2026-06-05 15:16:14 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-05 15:16:14 +0900 |
| commit | 20c6c675a11b887c603be5ff08165cb182c7db65 (patch) | |
| tree | e57ef71daa6cb7c5c73a4bd9af5be6cafe9d2158 /packages/trace-store/src | |
| parent | 4d94c530406567791dbe4ab06c838a83c2e26023 (diff) | |
| download | dispatch-20c6c675a11b887c603be5ff08165cb182c7db65.tar.gz dispatch-20c6c675a11b887c603be5ff08165cb182c7db65.zip | |
feat(observability): Phase B — trace-store (SQLite) + out-of-process collector + trace CLI (345 tests)
trace-store (bun:sqlite): records+bodies schema (thin/fat split), idempotent insertRecords (FNV-1a id + INSERT OR IGNORE), getTurn/getBody, pure renderEasyView (D8 timeline skeleton), trace CLI. Its own DB, isolated from storage-sqlite.
observability-collector: out-of-process bin — tail journal -> splitLines/drainOnce -> trace-store.insertRecords; offset sidecar; at-least-once + idempotent; fail-safe; clean SIGINT/SIGTERM drain.
Build-config (orchestrator): root tsconfig refs; both excluded from vitest + added to test:bun (bun:sqlite); bun install.
Verified: tsc -b clean, 345 tests (273 vitest + 72 bun), biome 0 warnings/0 infos. Pipeline proven end-to-end: app -> journal -> collector -> SQLite -> 'trace <turnId>' easy-view. Known follow-up (next commit): kernel spans are flat (parent=ROOT) — run-turn nesting fix.
Diffstat (limited to 'packages/trace-store/src')
| -rw-r--r-- | packages/trace-store/src/cli.ts | 16 | ||||
| -rw-r--r-- | packages/trace-store/src/easy-view.test.ts | 360 | ||||
| -rw-r--r-- | packages/trace-store/src/easy-view.ts | 205 | ||||
| -rw-r--r-- | packages/trace-store/src/index.ts | 3 | ||||
| -rw-r--r-- | packages/trace-store/src/store.test.ts | 224 | ||||
| -rw-r--r-- | packages/trace-store/src/store.ts | 254 |
6 files changed, 1062 insertions, 0 deletions
diff --git a/packages/trace-store/src/cli.ts b/packages/trace-store/src/cli.ts new file mode 100644 index 0000000..9092c91 --- /dev/null +++ b/packages/trace-store/src/cli.ts @@ -0,0 +1,16 @@ +import { createTraceStore } from "./store.js"; + +const turnId = process.argv[2]; +if (turnId === undefined) { + console.error("Usage: bun packages/trace-store/src/cli.ts <turnId> [dbPath]"); + process.exit(1); +} + +const dbPath = process.argv[3] ?? process.env.TRACE_DB_PATH ?? "./.dispatch-data/traces.db"; +const store = createTraceStore({ path: dbPath }); +try { + const output = store.easyView(turnId); + console.log(output); +} finally { + store.close(); +} diff --git a/packages/trace-store/src/easy-view.test.ts b/packages/trace-store/src/easy-view.test.ts new file mode 100644 index 0000000..3ebc814 --- /dev/null +++ b/packages/trace-store/src/easy-view.test.ts @@ -0,0 +1,360 @@ +import type { LogRecord } from "@dispatch/kernel"; +import { describe, expect, it } from "vitest"; +import { formatDuration, renderEasyView } from "./easy-view.js"; + +describe("renderEasyView", () => { + it("returns empty string for empty records", () => { + expect(renderEasyView([])).toBe(""); + }); + + it("renders a single span with no children", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "s1", + name: "step", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + }, + { + kind: "span-close", + spanId: "s1", + name: "step", + timestamp: 2800, + durationMs: 1800, + status: "ok", + extensionId: "ext", + turnId: "t1", + }, + ]; + expect(renderEasyView(records)).toBe("- step 1.8s"); + }); + + it("renders a span with nested log records", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "s1", + name: "step", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + }, + { + kind: "log", + level: "info", + msg: "thinking...", + timestamp: 1200, + extensionId: "ext", + turnId: "t1", + parentSpanId: "s1", + }, + { + kind: "span-close", + spanId: "s1", + name: "step", + timestamp: 2800, + durationMs: 1800, + status: "ok", + extensionId: "ext", + turnId: "t1", + }, + ]; + const expected = ["- step 1.8s", " - thinking..."].join("\n"); + expect(renderEasyView(records)).toBe(expected); + }); + + it("renders nested spans with indentation", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "parent", + name: "step", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + }, + { + kind: "span-open", + spanId: "child", + name: "tool.call", + timestamp: 1100, + extensionId: "ext", + turnId: "t1", + parentSpanId: "parent", + }, + { + kind: "log", + level: "info", + msg: "executing", + timestamp: 1200, + extensionId: "ext", + turnId: "t1", + parentSpanId: "child", + }, + { + kind: "span-close", + spanId: "child", + name: "tool.call", + timestamp: 1500, + durationMs: 400, + status: "ok", + extensionId: "ext", + turnId: "t1", + parentSpanId: "parent", + }, + { + kind: "span-close", + spanId: "parent", + name: "step", + timestamp: 2800, + durationMs: 1800, + status: "ok", + extensionId: "ext", + turnId: "t1", + }, + ]; + const expected = ["- step 1.8s", " - tool.call 400ms", " - executing"].join("\n"); + expect(renderEasyView(records)).toBe(expected); + }); + + it("renders a span-open without matching span-close as (open)", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "s-open", + name: "step", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + }, + ]; + expect(renderEasyView(records)).toBe("- step (open)"); + }); + + it("renders root-level log records without a parent span", () => { + const records: LogRecord[] = [ + { + kind: "log", + level: "info", + msg: "top-level", + timestamp: 500, + extensionId: "ext", + turnId: "t1", + }, + { + kind: "log", + level: "warn", + msg: "a warning", + timestamp: 600, + extensionId: "ext", + turnId: "t1", + }, + ]; + const expected = ["- top-level", "- [warn] a warning"].join("\n"); + expect(renderEasyView(records)).toBe(expected); + }); + + it("skips span-close records without a matching span-open", () => { + const records: LogRecord[] = [ + { + kind: "span-close", + spanId: "orphan", + name: "orphan", + timestamp: 1000, + durationMs: 100, + status: "ok", + extensionId: "ext", + turnId: "t1", + }, + { + kind: "log", + level: "info", + msg: "real", + timestamp: 2000, + extensionId: "ext", + turnId: "t1", + }, + ]; + expect(renderEasyView(records)).toBe("- real"); + }); + + it("renders span with ERR status", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "s1", + name: "failing-step", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + }, + { + kind: "span-close", + spanId: "s1", + name: "failing-step", + timestamp: 1100, + durationMs: 100, + status: "error", + extensionId: "ext", + turnId: "t1", + }, + ]; + expect(renderEasyView(records)).toBe("- failing-step 100ms ERR"); + }); + + it("renders body hint when body is present", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "s1", + name: "prompt", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + body: "x".repeat(2100), + }, + { + kind: "span-close", + spanId: "s1", + name: "prompt", + timestamp: 1100, + durationMs: 100, + status: "ok", + extensionId: "ext", + turnId: "t1", + }, + ]; + expect(renderEasyView(records)).toBe("- prompt 100ms [body 2.1k]"); + }); + + it("renders log record body hint", () => { + const records: LogRecord[] = [ + { + kind: "log", + level: "debug", + msg: "payload", + timestamp: 500, + extensionId: "ext", + turnId: "t1", + body: "abc", + }, + ]; + expect(renderEasyView(records)).toBe("- [debug] payload [body 3b]"); + }); + + it("renders attributes on span (up to 3)", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "s1", + name: "provider.request", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + attributes: { model: "gpt-4", method: "POST", status: 200 }, + }, + { + kind: "span-close", + spanId: "s1", + name: "provider.request", + timestamp: 1500, + durationMs: 500, + status: "ok", + extensionId: "ext", + turnId: "t1", + }, + ]; + const output = renderEasyView(records); + expect(output).toContain("provider.request 500ms"); + expect(output).toContain('model="gpt-4"'); + expect(output).toContain('method="POST"'); + expect(output).toContain("status=200"); + }); + + it("renders mixed records in correct order", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "s1", + name: "step", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + }, + { + kind: "log", + level: "info", + msg: "inside step", + timestamp: 1200, + extensionId: "ext", + turnId: "t1", + parentSpanId: "s1", + }, + { + kind: "span-close", + spanId: "s1", + name: "step", + timestamp: 2800, + durationMs: 1800, + status: "ok", + extensionId: "ext", + turnId: "t1", + }, + { + kind: "log", + level: "info", + msg: "after step", + timestamp: 3000, + extensionId: "ext", + turnId: "t1", + }, + ]; + const expected = ["- step 1.8s", " - inside step", "- after step"].join("\n"); + expect(renderEasyView(records)).toBe(expected); + }); + + it("deterministic output for same input", () => { + const records: LogRecord[] = [ + { + kind: "span-open", + spanId: "s1", + name: "step", + timestamp: 1000, + extensionId: "ext", + turnId: "t1", + }, + { + kind: "span-close", + spanId: "s1", + name: "step", + timestamp: 1800, + durationMs: 800, + status: "ok", + extensionId: "ext", + turnId: "t1", + }, + ]; + expect(renderEasyView(records)).toBe(renderEasyView(records)); + }); +}); + +describe("formatDuration", () => { + it("formats milliseconds under 1s", () => { + expect(formatDuration(42)).toBe("42ms"); + expect(formatDuration(999)).toBe("999ms"); + }); + + it("formats seconds", () => { + expect(formatDuration(1000)).toBe("1.0s"); + expect(formatDuration(1500)).toBe("1.5s"); + expect(formatDuration(59999)).toBe("60.0s"); + }); + + it("formats minutes", () => { + expect(formatDuration(60000)).toBe("1m0s"); + expect(formatDuration(90000)).toBe("1m30s"); + expect(formatDuration(125000)).toBe("2m5s"); + }); +}); diff --git a/packages/trace-store/src/easy-view.ts b/packages/trace-store/src/easy-view.ts new file mode 100644 index 0000000..55477ee --- /dev/null +++ b/packages/trace-store/src/easy-view.ts @@ -0,0 +1,205 @@ +import type { Attributes, LogRecord } from "@dispatch/kernel"; + +interface SpanInfo { + name: string; + openTimestamp: number; + closeTimestamp?: number; + durationMs?: number; + status?: string; + body?: string; + attributes?: Attributes; + children: TimelineEntry[]; +} + +interface LogEntry { + record: LogRecord; +} + +type TimelineEntry = { type: "span"; span: SpanInfo } | { type: "log"; entry: LogEntry }; + +export function renderEasyView(records: readonly LogRecord[]): string { + if (records.length === 0) { + return ""; + } + + const sorted = [...records].sort((a, b) => a.timestamp - b.timestamp); + + const spansById = new Map<string, SpanInfo>(); + const roots: TimelineEntry[] = []; + const childrenByParent = new Map<string | undefined, TimelineEntry[]>(); + + for (const r of sorted) { + if (r.kind === "span-open") { + const span: SpanInfo = { + name: r.name, + openTimestamp: r.timestamp, + ...(r.body !== undefined && { body: r.body }), + ...(r.attributes !== undefined && { attributes: r.attributes }), + children: [], + }; + spansById.set(r.spanId, span); + } else if (r.kind === "span-close") { + const span = spansById.get(r.spanId); + if (span !== undefined) { + span.closeTimestamp = r.timestamp; + span.durationMs = r.durationMs; + span.status = r.status; + if (r.body !== undefined) { + span.body = r.body; + } + if (r.attributes !== undefined) { + span.attributes = { ...span.attributes, ...r.attributes }; + } + } + } else { + const entry: TimelineEntry = { type: "log", entry: { record: r } }; + const parentId = r.parentSpanId; + let siblings = childrenByParent.get(parentId); + if (siblings === undefined) { + siblings = []; + childrenByParent.set(parentId, siblings); + } + siblings.push(entry); + } + } + + for (const [spanId, span] of spansById) { + const entry: TimelineEntry = { type: "span", span }; + const parentId = getSpanParentId(sorted, spanId); + let siblings = childrenByParent.get(parentId); + if (siblings === undefined) { + siblings = []; + childrenByParent.set(parentId, siblings); + } + siblings.push(entry); + } + + for (const [parentId, children] of childrenByParent) { + if (parentId === undefined) { + roots.push(...children); + } else { + const parent = spansById.get(parentId); + if (parent !== undefined) { + parent.children.push(...children); + } else { + roots.push(...children); + } + } + } + + sortByTimestamp(roots); + for (const entry of roots) { + if (entry.type === "span") { + sortByTimestamp(entry.span.children); + } + } + + const lines: string[] = []; + for (const entry of roots) { + renderEntry(entry, 0, lines); + } + return lines.join("\n"); +} + +function sortByTimestamp(entries: TimelineEntry[]): void { + entries.sort((a, b) => { + const tsA = a.type === "span" ? a.span.openTimestamp : a.entry.record.timestamp; + const tsB = b.type === "span" ? b.span.openTimestamp : b.entry.record.timestamp; + return tsA - tsB; + }); +} + +function getSpanParentId(records: readonly LogRecord[], spanId: string): string | undefined { + for (const r of records) { + if (r.kind === "span-open" && r.spanId === spanId) { + return r.parentSpanId; + } + } + return undefined; +} + +function renderEntry(entry: TimelineEntry, depth: number, lines: string[]): void { + const indent = " ".repeat(depth); + + if (entry.type === "span") { + const span = entry.span; + const dur = span.durationMs !== undefined ? formatDuration(span.durationMs) : undefined; + const statusStr = span.status === "error" ? " ERR" : ""; + const suffix = dur !== undefined ? ` ${dur}${statusStr}` : " (open)"; + + let detail = ""; + if (span.attributes !== undefined) { + const keys = Object.keys(span.attributes); + if (keys.length > 0) { + const parts: string[] = []; + for (const k of keys.slice(0, 3)) { + const v = span.attributes[k]; + if (v !== undefined) { + parts.push(`${k}=${formatAttrValue(v)}`); + } + } + detail = ` {${parts.join(", ")}}`; + } + } + + const bodyHint = span.body !== undefined ? ` [body ${formatSize(span.body.length)}]` : ""; + + lines.push(`${indent}- ${span.name}${suffix}${detail}${bodyHint}`); + + for (const child of span.children) { + renderEntry(child, depth + 1, lines); + } + } else { + const r = entry.entry.record; + if (r.kind === "log") { + const lvl = levelTag(r.level); + const bodyHint = r.body !== undefined ? ` [body ${formatSize(r.body.length)}]` : ""; + lines.push(`${indent}- ${lvl}${r.msg}${bodyHint}`); + } + } +} + +function levelTag(level: string): string { + if (level === "info") { + return ""; + } + return `[${level}] `; +} + +export function formatDuration(ms: number): string { + if (ms < 1000) { + return `${ms}ms`; + } + const s = ms / 1000; + if (s < 60) { + return `${s.toFixed(1)}s`; + } + const m = Math.floor(s / 60); + const rem = (s % 60).toFixed(0); + return `${m}m${rem}s`; +} + +function formatSize(n: number): string { + if (n < 1024) { + return `${n}b`; + } + const kb = n / 1024; + if (kb < 1024) { + return `${kb.toFixed(1)}k`; + } + const mb = kb / 1024; + return `${mb.toFixed(1)}M`; +} + +function formatAttrValue(value: string | number | boolean | null): string { + if (value === null) { + return "null"; + } + if (typeof value === "string") { + if (value.length > 20) { + return `"${value.slice(0, 17)}..."`; + } + return `"${value}"`; + } + return String(value); +} diff --git a/packages/trace-store/src/index.ts b/packages/trace-store/src/index.ts new file mode 100644 index 0000000..7341305 --- /dev/null +++ b/packages/trace-store/src/index.ts @@ -0,0 +1,3 @@ +export { formatDuration, renderEasyView } from "./easy-view.js"; +export type { TraceStore } from "./store.js"; +export { createTraceStore, stableId } from "./store.js"; diff --git a/packages/trace-store/src/store.test.ts b/packages/trace-store/src/store.test.ts new file mode 100644 index 0000000..e380147 --- /dev/null +++ b/packages/trace-store/src/store.test.ts @@ -0,0 +1,224 @@ +import type { LogRecord } from "@dispatch/kernel"; +import { describe, expect, it } from "vitest"; +import { createTraceStore, stableId } from "./store.js"; + +const logRecord: LogRecord = { + kind: "log", + level: "info", + msg: "hello", + timestamp: 1700000000000, + extensionId: "test-ext", + conversationId: "conv-1", + turnId: "turn-1", + spanId: "span-1", + attributes: { key: "value" }, +}; + +const spanOpenRecord: LogRecord = { + kind: "span-open", + spanId: "span-2", + name: "step", + timestamp: 1700000000100, + extensionId: "test-ext", + conversationId: "conv-1", + turnId: "turn-1", + parentSpanId: "span-1", +}; + +const spanCloseRecord: LogRecord = { + kind: "span-close", + spanId: "span-2", + name: "step", + timestamp: 1700000000500, + durationMs: 400, + status: "ok", + extensionId: "test-ext", + conversationId: "conv-1", + turnId: "turn-1", + parentSpanId: "span-1", +}; + +const bodyRecord: LogRecord = { + kind: "span-open", + spanId: "span-3", + name: "prompt", + timestamp: 1700000000200, + extensionId: "test-ext", + conversationId: "conv-1", + turnId: "turn-1", + body: "the full prompt text", +}; + +const logRecordNoBody: LogRecord = { + kind: "log", + level: "debug", + msg: "no body here", + timestamp: 1700000000050, + extensionId: "ext-min", + turnId: "turn-1", +}; + +describe("stableId", () => { + it("produces a 16-char hex string", () => { + const id = stableId(logRecord); + expect(id).toMatch(/^[0-9a-f]{16}$/); + }); + + it("is deterministic for the same record", () => { + expect(stableId(logRecord)).toBe(stableId(logRecord)); + }); + + it("produces different ids for different records", () => { + expect(stableId(logRecord)).not.toBe(stableId(spanOpenRecord)); + }); +}); + +describe("createTraceStore", () => { + function freshStore() { + return createTraceStore({ path: ":memory:" }); + } + + it("inserts and retrieves records ordered by timestamp", () => { + const store = freshStore(); + store.insertRecords([logRecord, spanCloseRecord, spanOpenRecord]); + const result = store.getTurn("turn-1"); + expect(result).toHaveLength(3); + expect(result[0]?.timestamp).toBe(1700000000000); + expect(result[1]?.timestamp).toBe(1700000000100); + expect(result[2]?.timestamp).toBe(1700000000500); + store.close(); + }); + + it("reconstructs attributes from JSON", () => { + const store = freshStore(); + store.insertRecords([logRecord]); + const result = store.getTurn("turn-1"); + expect(result[0]?.kind).toBe("log"); + if (result[0]?.kind === "log") { + expect(result[0].attributes).toEqual({ key: "value" }); + } + store.close(); + }); + + it("reconstructs links from JSON", () => { + const store = freshStore(); + const withLinks: LogRecord = { + kind: "span-open", + spanId: "span-link", + name: "linked", + timestamp: 1700000000999, + extensionId: "ext", + turnId: "turn-1", + links: [{ spanId: "other-span", turnId: "turn-0", reason: "caused" }], + }; + store.insertRecords([withLinks]); + const result = store.getTurn("turn-1"); + expect(result[0]?.kind).toBe("span-open"); + if (result[0]?.kind === "span-open") { + expect(result[0].links).toEqual([ + { spanId: "other-span", turnId: "turn-0", reason: "caused" }, + ]); + } + store.close(); + }); + + it("returns empty array for unknown turnId", () => { + const store = freshStore(); + store.insertRecords([logRecord]); + expect(store.getTurn("nonexistent")).toEqual([]); + store.close(); + }); + + it("getBody returns body for body-bearing records", () => { + const store = freshStore(); + store.insertRecords([bodyRecord]); + const id = stableId(bodyRecord); + expect(store.getBody(id)).toBe("the full prompt text"); + store.close(); + }); + + it("getBody returns undefined for records without body", () => { + const store = freshStore(); + store.insertRecords([logRecordNoBody]); + const id = stableId(logRecordNoBody); + expect(store.getBody(id)).toBeUndefined(); + store.close(); + }); + + it("bodies table only holds body-bearing records", () => { + const store = freshStore(); + store.insertRecords([logRecord, bodyRecord, logRecordNoBody]); + const bodyId = stableId(bodyRecord); + expect(store.getBody(bodyId)).toBe("the full prompt text"); + + const logId = stableId(logRecord); + expect(store.getBody(logId)).toBeUndefined(); + + const noBodyId = stableId(logRecordNoBody); + expect(store.getBody(noBodyId)).toBeUndefined(); + store.close(); + }); + + it("is idempotent — re-inserting the same records produces no duplicates", () => { + const store = freshStore(); + store.insertRecords([logRecord, spanOpenRecord, spanCloseRecord, bodyRecord]); + store.insertRecords([logRecord, spanOpenRecord, spanCloseRecord, bodyRecord]); + const result = store.getTurn("turn-1"); + expect(result).toHaveLength(4); + store.close(); + }); + + it("handles span-close record with attributes and links round-trip", () => { + const store = freshStore(); + const closeWithMeta: LogRecord = { + kind: "span-close", + spanId: "span-m", + name: "step", + timestamp: 1700000001000, + durationMs: 250, + status: "error", + extensionId: "ext", + turnId: "turn-1", + attributes: { httpStatus: 500 }, + links: [{ spanId: "upstream" }], + }; + store.insertRecords([closeWithMeta]); + const result = store.getTurn("turn-1"); + expect(result[0]?.kind).toBe("span-close"); + if (result[0]?.kind === "span-close") { + expect(result[0].durationMs).toBe(250); + expect(result[0].status).toBe("error"); + expect(result[0].attributes).toEqual({ httpStatus: 500 }); + expect(result[0].links).toEqual([{ spanId: "upstream" }]); + } + store.close(); + }); + + it("easyView delegates to renderEasyView", () => { + const store = freshStore(); + store.insertRecords([spanOpenRecord]); + const output = store.easyView("turn-1"); + expect(output).toContain("step (open)"); + store.close(); + }); + + it("persists to a file path", () => { + const tmpPath = `/tmp/trace-store-test-${Date.now()}.db`; + const store = createTraceStore({ path: tmpPath }); + store.insertRecords([logRecord]); + store.close(); + + const store2 = createTraceStore({ path: tmpPath }); + const result = store2.getTurn("turn-1"); + expect(result).toHaveLength(1); + expect(result[0]?.kind).toBe("log"); + store2.close(); + + const { unlinkSync } = require("node:fs"); + try { + unlinkSync(tmpPath); + } catch { + // ignore cleanup error + } + }); +}); diff --git a/packages/trace-store/src/store.ts b/packages/trace-store/src/store.ts new file mode 100644 index 0000000..53b7390 --- /dev/null +++ b/packages/trace-store/src/store.ts @@ -0,0 +1,254 @@ +import { Database } from "bun:sqlite"; +import type { Attributes, LogRecord, SpanLink } from "@dispatch/kernel"; +import { renderEasyView } from "./easy-view.js"; + +export interface TraceStore { + insertRecords(records: readonly LogRecord[]): void; + getTurn(turnId: string): LogRecord[]; + getBody(recordId: string): string | undefined; + easyView(turnId: string): string; + close(): void; +} + +export function createTraceStore(opts: { path: string }): TraceStore { + const db = new Database(opts.path); + db.run("PRAGMA journal_mode = WAL"); + schema(db); + return { + insertRecords(records) { + insertRecords(db, records); + }, + getTurn(turnId) { + return getTurn(db, turnId); + }, + getBody(recordId) { + return getBody(db, recordId); + }, + easyView(turnId) { + return renderEasyView(getTurn(db, turnId)); + }, + close() { + db.close(); + }, + }; +} + +function schema(db: Database): void { + db.run(` + CREATE TABLE IF NOT EXISTS records ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + level TEXT, + msg TEXT, + name TEXT, + spanId TEXT, + parentSpanId TEXT, + conversationId TEXT, + turnId TEXT, + extensionId TEXT NOT NULL, + timestamp INTEGER NOT NULL, + durationMs INTEGER, + status TEXT, + attributes TEXT, + links TEXT + ) + `); + db.run("CREATE INDEX IF NOT EXISTS idx_records_turnId ON records(turnId)"); + db.run("CREATE INDEX IF NOT EXISTS idx_records_conversationId ON records(conversationId)"); + db.run("CREATE INDEX IF NOT EXISTS idx_records_spanId ON records(spanId)"); + db.run("CREATE INDEX IF NOT EXISTS idx_records_kind ON records(kind)"); + db.run("CREATE INDEX IF NOT EXISTS idx_records_timestamp ON records(timestamp)"); + + db.run(` + CREATE TABLE IF NOT EXISTS bodies ( + recordId TEXT PRIMARY KEY REFERENCES records(id), + body TEXT NOT NULL + ) + `); +} + +function insertRecords(db: Database, records: readonly LogRecord[]): void { + const recStmt = db.prepare(` + INSERT OR IGNORE INTO records + (id, kind, level, msg, name, spanId, parentSpanId, + conversationId, turnId, extensionId, timestamp, + durationMs, status, attributes, links) + VALUES + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const bodyStmt = db.prepare("INSERT OR IGNORE INTO bodies (recordId, body) VALUES (?, ?)"); + + const txn = db.transaction(() => { + for (const r of records) { + const id = stableId(r); + const kind = r.kind; + let level: string | null = null; + let msg: string | null = null; + let name: string | null = null; + let spanId: string | null = null; + let parentSpanId: string | null = null; + let durationMs: number | null = null; + let status: string | null = null; + let links: string | null = null; + + if (r.kind === "log") { + level = r.level; + msg = r.msg; + spanId = r.spanId ?? null; + parentSpanId = r.parentSpanId ?? null; + } else if (r.kind === "span-open") { + name = r.name; + spanId = r.spanId; + parentSpanId = r.parentSpanId ?? null; + if (r.links !== undefined) { + links = JSON.stringify(r.links); + } + } else { + name = r.name; + spanId = r.spanId; + parentSpanId = r.parentSpanId ?? null; + durationMs = r.durationMs; + status = r.status; + if (r.links !== undefined) { + links = JSON.stringify(r.links); + } + } + + const attributes: string | null = + r.attributes !== undefined ? JSON.stringify(r.attributes) : null; + + recStmt.run( + id, + kind, + level, + msg, + name, + spanId, + parentSpanId, + r.conversationId ?? null, + r.turnId ?? null, + r.extensionId, + r.timestamp, + durationMs, + status, + attributes, + links, + ); + + if (r.body !== undefined) { + bodyStmt.run(id, r.body); + } + } + }); + txn(); +} + +interface RecordRow { + id: string; + kind: string; + level: string | null; + msg: string | null; + name: string | null; + spanId: string | null; + parentSpanId: string | null; + conversationId: string | null; + turnId: string | null; + extensionId: string; + timestamp: number; + durationMs: number | null; + status: string | null; + attributes: string | null; + links: string | null; +} + +function getTurn(db: Database, turnId: string): LogRecord[] { + const rows = db + .query("SELECT * FROM records WHERE turnId = ? ORDER BY timestamp ASC, rowid ASC") + .all(turnId) as RecordRow[]; + return rows.map(rowToRecord); +} + +function getBody(db: Database, recordId: string): string | undefined { + const row = db.query("SELECT body FROM bodies WHERE recordId = ?").get(recordId) as { + body: string; + } | null; + return row?.body; +} + +function rowToRecord(row: RecordRow): LogRecord { + const attributes: Attributes | undefined = + row.attributes !== null ? JSON.parse(row.attributes) : undefined; + const links: SpanLink[] | undefined = row.links !== null ? JSON.parse(row.links) : undefined; + + if (row.kind === "log") { + const record: LogRecord = { + kind: "log", + level: row.level as "debug" | "info" | "warn" | "error", + msg: row.msg ?? "", + timestamp: row.timestamp, + extensionId: row.extensionId, + ...(row.conversationId !== null && { conversationId: row.conversationId }), + ...(row.turnId !== null && { turnId: row.turnId }), + ...(row.spanId !== null && { spanId: row.spanId }), + ...(row.parentSpanId !== null && { parentSpanId: row.parentSpanId }), + ...(attributes !== undefined && { attributes }), + }; + return record; + } + + if (row.kind === "span-open") { + const record: LogRecord = { + kind: "span-open", + spanId: row.spanId ?? "", + name: row.name ?? "", + timestamp: row.timestamp, + extensionId: row.extensionId, + ...(row.conversationId !== null && { conversationId: row.conversationId }), + ...(row.turnId !== null && { turnId: row.turnId }), + ...(row.parentSpanId !== null && { parentSpanId: row.parentSpanId }), + ...(attributes !== undefined && { attributes }), + ...(links !== undefined && { links }), + }; + return record; + } + + const record: LogRecord = { + kind: "span-close", + spanId: row.spanId ?? "", + name: row.name ?? "", + timestamp: row.timestamp, + durationMs: row.durationMs ?? 0, + status: (row.status as "ok" | "error") ?? "ok", + extensionId: row.extensionId, + ...(row.conversationId !== null && { conversationId: row.conversationId }), + ...(row.turnId !== null && { turnId: row.turnId }), + ...(row.parentSpanId !== null && { parentSpanId: row.parentSpanId }), + ...(attributes !== undefined && { attributes }), + ...(links !== undefined && { links }), + }; + return record; +} + +function toCanonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") { + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map(toCanonicalJson).join(",")}]`; + } + const obj = value as Record<string, unknown>; + const keys = Object.keys(obj).sort(); + const entries = keys.map((k) => `${JSON.stringify(k)}:${toCanonicalJson(obj[k])}`); + return `{${entries.join(",")}}`; +} + +export function stableId(record: LogRecord): string { + const json = toCanonicalJson(record); + let hash = 0xcbf29ce484222325n; + const prime = 0x100000001b3n; + for (let i = 0; i < json.length; i++) { + hash ^= BigInt(json.charCodeAt(i)); + hash = (hash * prime) & 0xffffffffffffffffn; + } + return hash.toString(16).padStart(16, "0"); +} |
