diff options
| author | Adam Malczewski <[email protected]> | 2026-06-25 18:36:08 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-25 18:36:08 +0900 |
| commit | de022cee7ac66c95d7ed6a35d4e00f8e2d92cbbc (patch) | |
| tree | 041dcb1017e544a405526443cb578baa974bec0e /packages/session-orchestrator/src | |
| parent | fc1c3a54c3075990ec0dd0f97901bd46fe142923 (diff) | |
| parent | 649fc4f66f40f7743683546f81d3320e7394e597 (diff) | |
| download | dispatch-de022cee7ac66c95d7ed6a35d4e00f8e2d92cbbc.tar.gz dispatch-de022cee7ac66c95d7ed6a35d4e00f8e2d92cbbc.zip | |
Merge branch 'dev' into feature/ssh-support
Brings dev's retry-with-backoff (the transient `provider-retry` AgentEvent the
web frontend consumes) + the LSP-dead-server per-edit-hang fix into the SSH
feature branch, alongside the SSH waves 0-5c.
All code files auto-merged cleanly (run-turn.ts, orchestrator.ts, runtime.ts,
wire/index.ts, tool-edit-file/extension.ts, run-turn.test.ts — both computerId
threading and retry-with-backoff coexist). Only tasks.md conflicted (status
section — orchestrator-resolved; both feature sections kept).
Verified post-merge: tsc -b EXIT 0, biome clean (391 files), 1730 vitest pass
+6 sshd-integration skipped (was 1690; +40 from dev's retry/LSP tests).
Wire dist rebuilt so the FE can re-sync the pinned @dispatch/wire dep and pick
up BOTH provider-retry AND the SSH Computer/defaultComputerId types.
No merge or push (into dev or otherwise).
Diffstat (limited to 'packages/session-orchestrator/src')
| -rw-r--r-- | packages/session-orchestrator/src/index.ts | 6 | ||||
| -rw-r--r-- | packages/session-orchestrator/src/orchestrator.ts | 36 | ||||
| -rw-r--r-- | packages/session-orchestrator/src/pure.test.ts | 61 | ||||
| -rw-r--r-- | packages/session-orchestrator/src/pure.ts | 47 |
4 files changed, 150 insertions, 0 deletions
diff --git a/packages/session-orchestrator/src/index.ts b/packages/session-orchestrator/src/index.ts index fa8d9e9..aaafb76 100644 --- a/packages/session-orchestrator/src/index.ts +++ b/packages/session-orchestrator/src/index.ts @@ -12,6 +12,7 @@ export { conversationOpened, conversationStatusChanged, createCompactionService, + createRetryStrategy, createSessionOrchestrator, createWarmService, type EnqueueInput, @@ -34,8 +35,13 @@ export { } from "./orchestrator.js"; export { buildUserMessage, + cumulativeSleepMs, defaultDispatchPolicy, + delayFor, generateTurnId, + RETRY_BUDGET_MS, + RETRY_SCHEDULE_MS, + RETRY_TAIL_MS, resolveReasoningEffort, selectFirstProvider, } from "./pure.js"; diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts index ae27e59..a1401d6 100644 --- a/packages/session-orchestrator/src/orchestrator.ts +++ b/packages/session-orchestrator/src/orchestrator.ts @@ -11,6 +11,7 @@ import type { ProviderEvent, ProviderStreamOptions, ReasoningEffort, + RetryStrategy, RunTurnInput, RunTurnResult, ToolContract, @@ -24,6 +25,7 @@ import { createMetricsAccumulator } from "./metrics.js"; import { buildUserMessage, defaultDispatchPolicy, + delayFor, generateTurnId, resolveModelName, resolveReasoningEffort, @@ -342,12 +344,45 @@ export interface SessionOrchestratorBundle { readonly activeConversations: ReadonlySet<string>; } +/** + * The concrete retry strategy wired into every turn's `RunTurnInput.retry`. + * + * `delayFor` is the pure schedule (`5s, 10s, 30s, 60s, 5m, 10m, 15m, 30m`, + * then repeat 30m until 8h cumulative scheduled sleep) — no I/O, no clock. + * `sleep` is the abortable I/O effect: a `setTimeout`-based promise that + * rejects when the turn's abort signal fires (so a retry in flight seals the + * turn `aborted`). The kernel imports no timer; this is the shell-provided I/O. + */ +export function createRetryStrategy(): RetryStrategy { + const sleep = (ms: number, signal: AbortSignal): Promise<void> => { + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(new Error("aborted")); + return; + } + const timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(new Error("aborted")); + }; + signal.addEventListener("abort", onAbort, { once: true }); + }); + }; + return { delayFor, sleep }; +} + export function createSessionOrchestrator( deps: SessionOrchestratorDeps, ): SessionOrchestratorBundle { const activeConversations = new Set<string>(); const subscribers = new Map<string, Set<TurnEventListener>>(); const activeTurns = new Map<string, ActiveTurn>(); + // One stateless retry strategy shared by every turn (delayFor is pure; sleep + // is a stateless setTimeout closure). Wired into each RunTurnInput.retry. + const retryStrategy = createRetryStrategy(); function emitToHub(conversationId: string, event: AgentEvent): void { const turn = activeTurns.get(conversationId); @@ -640,6 +675,7 @@ export function createSessionOrchestrator( turnId, signal: controller.signal, providerOpts, + retry: retryStrategy, ...(turnLogger !== undefined ? { logger: turnLogger } : {}), ...(effectiveCwd !== undefined ? { cwd: effectiveCwd } : {}), ...(effectiveComputerId !== undefined ? { computerId: effectiveComputerId } : {}), diff --git a/packages/session-orchestrator/src/pure.test.ts b/packages/session-orchestrator/src/pure.test.ts index 9e5d3c4..2cbe15f 100644 --- a/packages/session-orchestrator/src/pure.test.ts +++ b/packages/session-orchestrator/src/pure.test.ts @@ -2,8 +2,13 @@ import type { ProviderContract } from "@dispatch/kernel"; import { describe, expect, it } from "vitest"; import { buildUserMessage, + cumulativeSleepMs, defaultDispatchPolicy, + delayFor, generateTurnId, + RETRY_BUDGET_MS, + RETRY_SCHEDULE_MS, + RETRY_TAIL_MS, resolveReasoningEffort, selectFirstProvider, } from "./pure.js"; @@ -100,3 +105,59 @@ describe("resolveReasoningEffort", () => { expect(resolveReasoningEffort(undefined, "max")).toBe("max"); }); }); + +describe("retry backoff schedule (delayFor)", () => { + it("emits the stepped head: 5s, 10s, 30s, 60s, 5m, 10m, 15m, 30m", () => { + expect(delayFor(0)).toBe(5_000); + expect(delayFor(1)).toBe(10_000); + expect(delayFor(2)).toBe(30_000); + expect(delayFor(3)).toBe(60_000); + expect(delayFor(4)).toBe(300_000); + expect(delayFor(5)).toBe(600_000); + expect(delayFor(6)).toBe(900_000); + expect(delayFor(7)).toBe(1_800_000); + }); + + it("repeats 30m after the head", () => { + expect(delayFor(8)).toBe(RETRY_TAIL_MS); + expect(delayFor(9)).toBe(RETRY_TAIL_MS); + expect(delayFor(20)).toBe(RETRY_TAIL_MS); + }); + + it("gives up (returns undefined) once cumulative sleep exceeds 8h", () => { + // Head sums to 3,705,000 ms; +1,800,000 per extra step. 8h = 28,800,000. + // attempt 20 cumulative = 3,705,000 + 13*1,800,000 = 27,105,000 (< 8h) → retry. + expect(delayFor(20)).toBe(RETRY_TAIL_MS); + // attempt 21 cumulative = 27,105,000 + 1,800,000 = 28,905,000 (> 8h) → stop. + expect(delayFor(21)).toBeUndefined(); + }); + + it("cumulativeSleepMs matches the sum of the schedule", () => { + expect(cumulativeSleepMs(0)).toBe(5_000); + expect(cumulativeSleepMs(1)).toBe(15_000); + expect(cumulativeSleepMs(7)).toBe(RETRY_SCHEDULE_MS.reduce((a, b) => a + b, 0)); + // 8h budget is 28,800,000 ms. + expect(RETRY_BUDGET_MS).toBe(8 * 60 * 60 * 1000); + // The last retry (attempt 20) keeps cumulative under budget. + expect(cumulativeSleepMs(20)).toBeLessThanOrEqual(RETRY_BUDGET_MS); + // The next (attempt 21) exceeds it. + expect(cumulativeSleepMs(21)).toBeGreaterThan(RETRY_BUDGET_MS); + }); + + it("the full schedule has 21 retries then stops", () => { + const schedule: number[] = []; + let attempt = 0; + while (true) { + const delay = delayFor(attempt); + if (delay === undefined) break; + schedule.push(delay); + attempt++; + } + expect(schedule).toHaveLength(21); + expect(schedule[0]).toBe(5_000); + expect(schedule.at(-1)).toBe(RETRY_TAIL_MS); + // 8 stepped head + 13 tail repeats. + expect(schedule.slice(0, 8)).toEqual([...RETRY_SCHEDULE_MS]); + expect(schedule.slice(8).every((d) => d === RETRY_TAIL_MS)).toBe(true); + }); +}); diff --git a/packages/session-orchestrator/src/pure.ts b/packages/session-orchestrator/src/pure.ts index 9a31e17..a028cbe 100644 --- a/packages/session-orchestrator/src/pure.ts +++ b/packages/session-orchestrator/src/pure.ts @@ -9,6 +9,53 @@ export function buildUserMessage(text: string): ChatMessage { return { role: "user", chunks: [{ type: "text", text }] }; } +// ── Provider-error retry backoff schedule ─────────────────────────────────── +// +// Pure, deterministic delay decision (no I/O, no clock) for retrying retryable +// provider errors (HTTP 429 / 5xx "overloaded"). The concrete `sleep` (I/O) +// is wired in the orchestrator; this owns only the policy. + +/** + * Stepped backoff schedule (ms): 5s, 10s, 30s, 60s, 5m, 10m, 15m, 30m. + * After the head is exhausted, {@link RETRY_TAIL_MS} (30m) repeats. + */ +export const RETRY_SCHEDULE_MS = [ + 5_000, 10_000, 30_000, 60_000, 300_000, 600_000, 900_000, 1_800_000, +] as const; + +/** Tail delay (ms) repeated after the stepped head: 30 minutes. */ +export const RETRY_TAIL_MS = 1_800_000; + +/** Cumulative scheduled-sleep budget (ms) after which retrying gives up: 8h. */ +export const RETRY_BUDGET_MS = 8 * 60 * 60 * 1000; + +/** + * Cumulative scheduled sleep through `attempt` (sum of delay[0..attempt]). + * Pure — no I/O, no clock. + */ +export function cumulativeSleepMs(attempt: number): number { + let sum = 0; + for (let i = 0; i <= attempt; i++) { + sum += i < RETRY_SCHEDULE_MS.length ? (RETRY_SCHEDULE_MS[i] ?? RETRY_TAIL_MS) : RETRY_TAIL_MS; + } + return sum; +} + +/** + * Pure, deterministic delay decision for the retry strategy: given the + * 0-based attempt index, return the delay in ms to sleep before the next + * retry, or `undefined` to stop (cumulative budget exhausted). No I/O, no + * clock — fully testable. Matches the plan's schedule: + * `5s, 10s, 30s, 60s, 5m, 10m, 15m, 30m`, then repeat 30m until 8h of + * cumulative scheduled sleep is reached, then give up. + */ +export function delayFor(attempt: number): number | undefined { + const scheduled = RETRY_SCHEDULE_MS[attempt]; + const delay = scheduled !== undefined ? scheduled : RETRY_TAIL_MS; + if (cumulativeSleepMs(attempt) > RETRY_BUDGET_MS) return undefined; // over budget → stop + return delay; +} + /** * Resolve the reasoning-effort level for a turn: * per-turn override → persisted per-conversation value → default `"high"`. |
