summaryrefslogtreecommitdiffhomepage
path: root/packages/session-orchestrator/src
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-25 18:36:08 +0900
committerAdam Malczewski <[email protected]>2026-06-25 18:36:08 +0900
commitde022cee7ac66c95d7ed6a35d4e00f8e2d92cbbc (patch)
tree041dcb1017e544a405526443cb578baa974bec0e /packages/session-orchestrator/src
parentfc1c3a54c3075990ec0dd0f97901bd46fe142923 (diff)
parent649fc4f66f40f7743683546f81d3320e7394e597 (diff)
downloaddispatch-de022cee7ac66c95d7ed6a35d4e00f8e2d92cbbc.tar.gz
dispatch-de022cee7ac66c95d7ed6a35d4e00f8e2d92cbbc.zip
Merge branch 'dev' into feature/ssh-support
Brings dev's retry-with-backoff (the transient `provider-retry` AgentEvent the web frontend consumes) + the LSP-dead-server per-edit-hang fix into the SSH feature branch, alongside the SSH waves 0-5c. All code files auto-merged cleanly (run-turn.ts, orchestrator.ts, runtime.ts, wire/index.ts, tool-edit-file/extension.ts, run-turn.test.ts — both computerId threading and retry-with-backoff coexist). Only tasks.md conflicted (status section — orchestrator-resolved; both feature sections kept). Verified post-merge: tsc -b EXIT 0, biome clean (391 files), 1730 vitest pass +6 sshd-integration skipped (was 1690; +40 from dev's retry/LSP tests). Wire dist rebuilt so the FE can re-sync the pinned @dispatch/wire dep and pick up BOTH provider-retry AND the SSH Computer/defaultComputerId types. No merge or push (into dev or otherwise).
Diffstat (limited to 'packages/session-orchestrator/src')
-rw-r--r--packages/session-orchestrator/src/index.ts6
-rw-r--r--packages/session-orchestrator/src/orchestrator.ts36
-rw-r--r--packages/session-orchestrator/src/pure.test.ts61
-rw-r--r--packages/session-orchestrator/src/pure.ts47
4 files changed, 150 insertions, 0 deletions
diff --git a/packages/session-orchestrator/src/index.ts b/packages/session-orchestrator/src/index.ts
index fa8d9e9..aaafb76 100644
--- a/packages/session-orchestrator/src/index.ts
+++ b/packages/session-orchestrator/src/index.ts
@@ -12,6 +12,7 @@ export {
conversationOpened,
conversationStatusChanged,
createCompactionService,
+ createRetryStrategy,
createSessionOrchestrator,
createWarmService,
type EnqueueInput,
@@ -34,8 +35,13 @@ export {
} from "./orchestrator.js";
export {
buildUserMessage,
+ cumulativeSleepMs,
defaultDispatchPolicy,
+ delayFor,
generateTurnId,
+ RETRY_BUDGET_MS,
+ RETRY_SCHEDULE_MS,
+ RETRY_TAIL_MS,
resolveReasoningEffort,
selectFirstProvider,
} from "./pure.js";
diff --git a/packages/session-orchestrator/src/orchestrator.ts b/packages/session-orchestrator/src/orchestrator.ts
index ae27e59..a1401d6 100644
--- a/packages/session-orchestrator/src/orchestrator.ts
+++ b/packages/session-orchestrator/src/orchestrator.ts
@@ -11,6 +11,7 @@ import type {
ProviderEvent,
ProviderStreamOptions,
ReasoningEffort,
+ RetryStrategy,
RunTurnInput,
RunTurnResult,
ToolContract,
@@ -24,6 +25,7 @@ import { createMetricsAccumulator } from "./metrics.js";
import {
buildUserMessage,
defaultDispatchPolicy,
+ delayFor,
generateTurnId,
resolveModelName,
resolveReasoningEffort,
@@ -342,12 +344,45 @@ export interface SessionOrchestratorBundle {
readonly activeConversations: ReadonlySet<string>;
}
+/**
+ * The concrete retry strategy wired into every turn's `RunTurnInput.retry`.
+ *
+ * `delayFor` is the pure schedule (`5s, 10s, 30s, 60s, 5m, 10m, 15m, 30m`,
+ * then repeat 30m until 8h cumulative scheduled sleep) — no I/O, no clock.
+ * `sleep` is the abortable I/O effect: a `setTimeout`-based promise that
+ * rejects when the turn's abort signal fires (so a retry in flight seals the
+ * turn `aborted`). The kernel imports no timer; this is the shell-provided I/O.
+ */
+export function createRetryStrategy(): RetryStrategy {
+ const sleep = (ms: number, signal: AbortSignal): Promise<void> => {
+ return new Promise((resolve, reject) => {
+ if (signal.aborted) {
+ reject(new Error("aborted"));
+ return;
+ }
+ const timer = setTimeout(() => {
+ signal.removeEventListener("abort", onAbort);
+ resolve();
+ }, ms);
+ const onAbort = () => {
+ clearTimeout(timer);
+ reject(new Error("aborted"));
+ };
+ signal.addEventListener("abort", onAbort, { once: true });
+ });
+ };
+ return { delayFor, sleep };
+}
+
export function createSessionOrchestrator(
deps: SessionOrchestratorDeps,
): SessionOrchestratorBundle {
const activeConversations = new Set<string>();
const subscribers = new Map<string, Set<TurnEventListener>>();
const activeTurns = new Map<string, ActiveTurn>();
+ // One stateless retry strategy shared by every turn (delayFor is pure; sleep
+ // is a stateless setTimeout closure). Wired into each RunTurnInput.retry.
+ const retryStrategy = createRetryStrategy();
function emitToHub(conversationId: string, event: AgentEvent): void {
const turn = activeTurns.get(conversationId);
@@ -640,6 +675,7 @@ export function createSessionOrchestrator(
turnId,
signal: controller.signal,
providerOpts,
+ retry: retryStrategy,
...(turnLogger !== undefined ? { logger: turnLogger } : {}),
...(effectiveCwd !== undefined ? { cwd: effectiveCwd } : {}),
...(effectiveComputerId !== undefined ? { computerId: effectiveComputerId } : {}),
diff --git a/packages/session-orchestrator/src/pure.test.ts b/packages/session-orchestrator/src/pure.test.ts
index 9e5d3c4..2cbe15f 100644
--- a/packages/session-orchestrator/src/pure.test.ts
+++ b/packages/session-orchestrator/src/pure.test.ts
@@ -2,8 +2,13 @@ import type { ProviderContract } from "@dispatch/kernel";
import { describe, expect, it } from "vitest";
import {
buildUserMessage,
+ cumulativeSleepMs,
defaultDispatchPolicy,
+ delayFor,
generateTurnId,
+ RETRY_BUDGET_MS,
+ RETRY_SCHEDULE_MS,
+ RETRY_TAIL_MS,
resolveReasoningEffort,
selectFirstProvider,
} from "./pure.js";
@@ -100,3 +105,59 @@ describe("resolveReasoningEffort", () => {
expect(resolveReasoningEffort(undefined, "max")).toBe("max");
});
});
+
+describe("retry backoff schedule (delayFor)", () => {
+ it("emits the stepped head: 5s, 10s, 30s, 60s, 5m, 10m, 15m, 30m", () => {
+ expect(delayFor(0)).toBe(5_000);
+ expect(delayFor(1)).toBe(10_000);
+ expect(delayFor(2)).toBe(30_000);
+ expect(delayFor(3)).toBe(60_000);
+ expect(delayFor(4)).toBe(300_000);
+ expect(delayFor(5)).toBe(600_000);
+ expect(delayFor(6)).toBe(900_000);
+ expect(delayFor(7)).toBe(1_800_000);
+ });
+
+ it("repeats 30m after the head", () => {
+ expect(delayFor(8)).toBe(RETRY_TAIL_MS);
+ expect(delayFor(9)).toBe(RETRY_TAIL_MS);
+ expect(delayFor(20)).toBe(RETRY_TAIL_MS);
+ });
+
+ it("gives up (returns undefined) once cumulative sleep exceeds 8h", () => {
+ // Head sums to 3,705,000 ms; +1,800,000 per extra step. 8h = 28,800,000.
+ // attempt 20 cumulative = 3,705,000 + 13*1,800,000 = 27,105,000 (< 8h) → retry.
+ expect(delayFor(20)).toBe(RETRY_TAIL_MS);
+ // attempt 21 cumulative = 27,105,000 + 1,800,000 = 28,905,000 (> 8h) → stop.
+ expect(delayFor(21)).toBeUndefined();
+ });
+
+ it("cumulativeSleepMs matches the sum of the schedule", () => {
+ expect(cumulativeSleepMs(0)).toBe(5_000);
+ expect(cumulativeSleepMs(1)).toBe(15_000);
+ expect(cumulativeSleepMs(7)).toBe(RETRY_SCHEDULE_MS.reduce((a, b) => a + b, 0));
+ // 8h budget is 28,800,000 ms.
+ expect(RETRY_BUDGET_MS).toBe(8 * 60 * 60 * 1000);
+ // The last retry (attempt 20) keeps cumulative under budget.
+ expect(cumulativeSleepMs(20)).toBeLessThanOrEqual(RETRY_BUDGET_MS);
+ // The next (attempt 21) exceeds it.
+ expect(cumulativeSleepMs(21)).toBeGreaterThan(RETRY_BUDGET_MS);
+ });
+
+ it("the full schedule has 21 retries then stops", () => {
+ const schedule: number[] = [];
+ let attempt = 0;
+ while (true) {
+ const delay = delayFor(attempt);
+ if (delay === undefined) break;
+ schedule.push(delay);
+ attempt++;
+ }
+ expect(schedule).toHaveLength(21);
+ expect(schedule[0]).toBe(5_000);
+ expect(schedule.at(-1)).toBe(RETRY_TAIL_MS);
+ // 8 stepped head + 13 tail repeats.
+ expect(schedule.slice(0, 8)).toEqual([...RETRY_SCHEDULE_MS]);
+ expect(schedule.slice(8).every((d) => d === RETRY_TAIL_MS)).toBe(true);
+ });
+});
diff --git a/packages/session-orchestrator/src/pure.ts b/packages/session-orchestrator/src/pure.ts
index 9a31e17..a028cbe 100644
--- a/packages/session-orchestrator/src/pure.ts
+++ b/packages/session-orchestrator/src/pure.ts
@@ -9,6 +9,53 @@ export function buildUserMessage(text: string): ChatMessage {
return { role: "user", chunks: [{ type: "text", text }] };
}
+// ── Provider-error retry backoff schedule ───────────────────────────────────
+//
+// Pure, deterministic delay decision (no I/O, no clock) for retrying retryable
+// provider errors (HTTP 429 / 5xx "overloaded"). The concrete `sleep` (I/O)
+// is wired in the orchestrator; this owns only the policy.
+
+/**
+ * Stepped backoff schedule (ms): 5s, 10s, 30s, 60s, 5m, 10m, 15m, 30m.
+ * After the head is exhausted, {@link RETRY_TAIL_MS} (30m) repeats.
+ */
+export const RETRY_SCHEDULE_MS = [
+ 5_000, 10_000, 30_000, 60_000, 300_000, 600_000, 900_000, 1_800_000,
+] as const;
+
+/** Tail delay (ms) repeated after the stepped head: 30 minutes. */
+export const RETRY_TAIL_MS = 1_800_000;
+
+/** Cumulative scheduled-sleep budget (ms) after which retrying gives up: 8h. */
+export const RETRY_BUDGET_MS = 8 * 60 * 60 * 1000;
+
+/**
+ * Cumulative scheduled sleep through `attempt` (sum of delay[0..attempt]).
+ * Pure — no I/O, no clock.
+ */
+export function cumulativeSleepMs(attempt: number): number {
+ let sum = 0;
+ for (let i = 0; i <= attempt; i++) {
+ sum += i < RETRY_SCHEDULE_MS.length ? (RETRY_SCHEDULE_MS[i] ?? RETRY_TAIL_MS) : RETRY_TAIL_MS;
+ }
+ return sum;
+}
+
+/**
+ * Pure, deterministic delay decision for the retry strategy: given the
+ * 0-based attempt index, return the delay in ms to sleep before the next
+ * retry, or `undefined` to stop (cumulative budget exhausted). No I/O, no
+ * clock — fully testable. Matches the plan's schedule:
+ * `5s, 10s, 30s, 60s, 5m, 10m, 15m, 30m`, then repeat 30m until 8h of
+ * cumulative scheduled sleep is reached, then give up.
+ */
+export function delayFor(attempt: number): number | undefined {
+ const scheduled = RETRY_SCHEDULE_MS[attempt];
+ const delay = scheduled !== undefined ? scheduled : RETRY_TAIL_MS;
+ if (cumulativeSleepMs(attempt) > RETRY_BUDGET_MS) return undefined; // over budget → stop
+ return delay;
+}
+
/**
* Resolve the reasoning-effort level for a turn:
* per-turn override → persisted per-conversation value → default `"high"`.