summaryrefslogtreecommitdiffhomepage
path: root/packages/provider-openai-compat/src/stream.ts
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-04 23:22:03 +0900
committerAdam Malczewski <[email protected]>2026-06-04 23:22:03 +0900
commit9b611d614d123462e50492d78202dae696b99aa2 (patch)
tree7e0c85b60dd0becddbb04c29fdf14855c10c090f /packages/provider-openai-compat/src/stream.ts
parent974ce6f46c25a522a42c6bd04fd62ce2d031aad5 (diff)
downloaddispatch-9b611d614d123462e50492d78202dae696b99aa2.tar.gz
dispatch-9b611d614d123462e50492d78202dae696b99aa2.zip
feat(core-ext): storage-sqlite, auth-apikey, provider-openai-compat
- storage-sqlite: bun:sqlite StorageNamespace backend + migrations (21 bun tests) - auth-apikey: pure resolver from env → ApiKeyCredentials (4 tests) - provider-openai-compat: OpenAI-compatible SSE stream → ProviderEvents - orchestrator fixes: provider imports (@dispatch/kernel), missing dep, exactOptionalPropertyTypes (omit-when-undefined), root tsconfig refs - vitest excludes storage-sqlite (bun:sqlite); test:bun runs it under bun
Diffstat (limited to 'packages/provider-openai-compat/src/stream.ts')
-rw-r--r--packages/provider-openai-compat/src/stream.ts202
1 files changed, 202 insertions, 0 deletions
diff --git a/packages/provider-openai-compat/src/stream.ts b/packages/provider-openai-compat/src/stream.ts
new file mode 100644
index 0000000..7021120
--- /dev/null
+++ b/packages/provider-openai-compat/src/stream.ts
@@ -0,0 +1,202 @@
+import type {
+ ChatMessage,
+ ProviderEvent,
+ ProviderStreamOptions,
+ ToolContract,
+} from "@dispatch/kernel";
+import { convertMessages, type OpenAIMessage } from "./convert-messages.js";
+import { convertTools, type OpenAITool } from "./convert-tools.js";
+
+export interface StreamConfig {
+ readonly baseURL: string;
+ readonly apiKey: string;
+ readonly model: string;
+}
+
+export async function* streamChat(
+ config: StreamConfig,
+ messages: readonly ChatMessage[],
+ tools: readonly ToolContract[],
+ opts?: ProviderStreamOptions,
+): AsyncIterable<ProviderEvent> {
+ const openaiMessages = convertMessages(messages);
+ const openaiTools = convertTools(tools);
+
+ const systemPrompt = opts?.systemPrompt;
+ const finalMessages: OpenAIMessage[] = systemPrompt
+ ? [{ role: "system", content: systemPrompt }, ...openaiMessages]
+ : openaiMessages;
+
+ const body: Record<string, unknown> = {
+ model: opts?.model ?? config.model,
+ messages: finalMessages,
+ stream: true,
+ };
+
+ if (openaiTools.length > 0) {
+ body.tools = openaiTools satisfies OpenAITool[];
+ }
+ if (opts?.temperature !== undefined) {
+ body.temperature = opts.temperature;
+ }
+ if (opts?.maxTokens !== undefined) {
+ body.max_tokens = opts.maxTokens;
+ }
+
+ let response: Response;
+ try {
+ response = await fetch(`${config.baseURL}/chat/completions`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${config.apiKey}`,
+ },
+ body: JSON.stringify(body),
+ });
+ } catch (err) {
+ yield {
+ type: "error",
+ message: err instanceof Error ? err.message : String(err),
+ retryable: true,
+ };
+ return;
+ }
+
+ if (!response.ok) {
+ const text = await response.text().catch(() => "unknown");
+ yield {
+ type: "error",
+ message: `HTTP ${response.status}: ${text}`,
+ code: String(response.status),
+ retryable: response.status >= 500 || response.status === 429,
+ };
+ return;
+ }
+
+ if (!response.body) {
+ yield { type: "error", message: "Response body is null" };
+ return;
+ }
+
+ yield* readSSEStream(response.body);
+}
+
+async function* readSSEStream(body: ReadableStream<Uint8Array>): AsyncIterable<ProviderEvent> {
+ const reader = body.getReader();
+ const decoder = new TextDecoder();
+ let buffer = "";
+ const toolCalls = new Map<number, { id: string; name: string; arguments: string }>();
+
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split("\n");
+ buffer = lines.pop() ?? "";
+
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (!trimmed.startsWith("data:")) continue;
+
+ const data = trimmed.slice(5).trim();
+ if (data === "[DONE]") return;
+
+ let chunk: Record<string, unknown>;
+ try {
+ chunk = JSON.parse(data);
+ } catch {
+ yield { type: "error", message: `Invalid JSON in SSE data: ${data}` };
+ continue;
+ }
+
+ const choices = chunk.choices as
+ | Array<{
+ delta: Record<string, unknown>;
+ finish_reason?: string | null;
+ }>
+ | undefined;
+
+ if (choices) {
+ for (const choice of choices) {
+ const delta = choice.delta;
+
+ if (typeof delta.content === "string" && delta.content) {
+ yield { type: "text-delta", delta: delta.content };
+ }
+
+ if (typeof delta.reasoning_content === "string" && delta.reasoning_content) {
+ yield { type: "reasoning-delta", delta: delta.reasoning_content };
+ }
+
+ const tcs = delta.tool_calls as
+ | Array<{
+ index: number;
+ id?: string;
+ function?: { name?: string; arguments?: string };
+ }>
+ | undefined;
+
+ if (tcs) {
+ for (const tc of tcs) {
+ const existing = toolCalls.get(tc.index);
+ if (existing) {
+ if (tc.function?.arguments) {
+ existing.arguments += tc.function.arguments;
+ }
+ } else {
+ toolCalls.set(tc.index, {
+ id: tc.id ?? "",
+ name: tc.function?.name ?? "",
+ arguments: tc.function?.arguments ?? "",
+ });
+ }
+ }
+ }
+
+ if (choice.finish_reason) {
+ const sortedIndices = [...toolCalls.keys()].sort((a, b) => a - b);
+ for (const idx of sortedIndices) {
+ const acc = toolCalls.get(idx);
+ if (!acc) continue;
+ let input: unknown;
+ try {
+ input = JSON.parse(acc.arguments);
+ } catch {
+ input = acc.arguments;
+ }
+ yield {
+ type: "tool-call",
+ toolCallId: acc.id,
+ toolName: acc.name,
+ input,
+ };
+ }
+ yield { type: "finish", reason: choice.finish_reason };
+ }
+ }
+ }
+
+ const usage = chunk.usage as
+ | {
+ prompt_tokens?: number;
+ completion_tokens?: number;
+ }
+ | undefined;
+
+ if (usage) {
+ yield {
+ type: "usage",
+ usage: {
+ inputTokens: usage.prompt_tokens ?? 0,
+ outputTokens: usage.completion_tokens ?? 0,
+ },
+ };
+ }
+ }
+ }
+ } finally {
+ reader.releaseLock();
+ }
+}