test(e2e): isolate prompt tests with per-worker backend (#20464)

3 files changed, 111 insertions, 99 deletions

diff --git a/packages/app/e2e/prompt/mock.ts b/packages/app/e2e/prompt/mock.ts
new file mode 100644
index 000000000..eb40a70cb
--- /dev/null
+++ b/packages/app/e2e/prompt/mock.ts
@@ -0,0 +1,46 @@
+import { createSdk } from "../utils"
+
+export const openaiModel = { providerID: "openai", modelID: "gpt-5.3-chat-latest" }
+
+type Hit = { body: Record<string, unknown> }
+
+export function bodyText(hit: Hit) {
+  return JSON.stringify(hit.body)
+}
+
+export function titleMatch(hit: Hit) {
+  return bodyText(hit).includes("Generate a title for this conversation")
+}
+
+export function promptMatch(token: string) {
+  return (hit: Hit) => bodyText(hit).includes(token)
+}
+
+export async function withMockOpenAI<T>(input: { serverUrl: string; llmUrl: string; fn: () => Promise<T> }) {
+  const sdk = createSdk(undefined, input.serverUrl)
+  const prev = await sdk.global.config.get().then((res) => res.data ?? {})
+
+  try {
+    await sdk.global.config.update({
+      config: {
+        ...prev,
+        model: `${openaiModel.providerID}/${openaiModel.modelID}`,
+        enabled_providers: ["openai"],
+        provider: {
+          ...prev.provider,
+          openai: {
+            ...prev.provider?.openai,
+            options: {
+              ...prev.provider?.openai?.options,
+              apiKey: "test-key",
+              baseURL: input.llmUrl,
+            },
+          },
+        },
+      },
+    })
+    return await input.fn()
+  } finally {
+    await sdk.global.config.update({ config: prev })
+  }
+}
diff --git a/packages/app/e2e/prompt/prompt-async.spec.ts b/packages/app/e2e/prompt/prompt-async.spec.ts
index 51fbc3e4a..99fa5f2d4 100644
--- a/packages/app/e2e/prompt/prompt-async.spec.ts
+++ b/packages/app/e2e/prompt/prompt-async.spec.ts
@@ -1,47 +1,53 @@
 import { test, expect } from "../fixtures"
 import { promptSelector } from "../selectors"
-import { cleanupSession, sessionIDFromUrl, withSession } from "../actions"
+import { assistantText, sessionIDFromUrl, withSession } from "../actions"
+import { openaiModel, promptMatch, withMockOpenAI } from "./mock"
 
 const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()
 
 // Regression test for Issue #12453: the synchronous POST /message endpoint holds
 // the connection open while the agent works, causing "Failed to fetch" over
 // VPN/Tailscale. The fix switches to POST /prompt_async which returns immediately.
-test("prompt succeeds when sync message endpoint is unreachable", async ({ page, sdk, gotoSession }) => {
+test("prompt succeeds when sync message endpoint is unreachable", async ({
+  page,
+  llm,
+  backend,
+  withBackendProject,
+}) => {
   test.setTimeout(120_000)
 
   // Simulate Tailscale/VPN killing the long-lived sync connection
   await page.route("**/session/*/message", (route) => route.abort("connectionfailed"))
 
-  await gotoSession()
+  await withMockOpenAI({
+    serverUrl: backend.url,
+    llmUrl: llm.url,
+    fn: async () => {
+      const token = `E2E_ASYNC_${Date.now()}`
+      await llm.textMatch(promptMatch(token), token)
 
-  const token = `E2E_ASYNC_${Date.now()}`
-  await page.locator(promptSelector).click()
-  await page.keyboard.type(`Reply with exactly: ${token}`)
-  await page.keyboard.press("Enter")
+      await withBackendProject(
+        async (project) => {
+          await page.locator(promptSelector).click()
+          await page.keyboard.type(`Reply with exactly: ${token}`)
+          await page.keyboard.press("Enter")
 
-  await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
-  const sessionID = sessionIDFromUrl(page.url())!
+          await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
+          const sessionID = sessionIDFromUrl(page.url())!
+          project.trackSession(sessionID)
 
-  try {
-    // Agent response arrives via SSE despite sync endpoint being dead
-    await expect
-      .poll(
-        async () => {
-          const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
-          return messages
-            .filter((m) => m.info.role === "assistant")
-            .flatMap((m) => m.parts)
-            .filter((p) => p.type === "text")
-            .map((p) => p.text)
-            .join("\n")
+          await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
+
+          await expect
+            .poll(() => assistantText(project.sdk, sessionID), { timeout: 90_000 })
+            .toContain(token)
+        },
+        {
+          model: openaiModel,
         },
-        { timeout: 90_000 },
       )
-      .toContain(token)
-  } finally {
-    await cleanupSession({ sdk, sessionID })
-  }
+    },
+  })
 })
 
 test("failed prompt send restores the composer input", async ({ page, sdk, gotoSession }) => {
diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.ts
index 1acf17f5b..e4545e97a 100644
--- a/packages/app/e2e/prompt/prompt.spec.ts
+++ b/packages/app/e2e/prompt/prompt.spec.ts
@@ -1,44 +1,9 @@
-import fs from "node:fs/promises"
-import path from "node:path"
 import { test, expect } from "../fixtures"
 import { promptSelector } from "../selectors"
-import { sessionIDFromUrl } from "../actions"
-import { createSdk } from "../utils"
+import { assistantText, sessionIDFromUrl } from "../actions"
+import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock"
 
-async function config(dir: string, url: string) {
-  await fs.writeFile(
-    path.join(dir, "opencode.json"),
-    JSON.stringify({
-      $schema: "https://opencode.ai/config.json",
-      enabled_providers: ["e2e-llm"],
-      provider: {
-        "e2e-llm": {
-          name: "E2E LLM",
-          npm: "@ai-sdk/openai-compatible",
-          env: [],
-          models: {
-            "test-model": {
-              name: "Test Model",
-              tool_call: true,
-              limit: { context: 128000, output: 32000 },
-            },
-          },
-          options: {
-            apiKey: "test-key",
-            baseURL: url,
-          },
-        },
-      },
-      agent: {
-        build: {
-          model: "e2e-llm/test-model",
-        },
-      },
-    }),
-  )
-}
-
-test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => {
+test("can send a prompt and receive a reply", async ({ page, llm, backend, withBackendProject }) => {
   test.setTimeout(120_000)
 
   const pageErrors: string[] = []
@@ -48,48 +13,43 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject })
   page.on("pageerror", onPageError)
 
   try {
-    await withProject(
-      async (project) => {
-        const sdk = createSdk(project.directory)
+    await withMockOpenAI({
+      serverUrl: backend.url,
+      llmUrl: llm.url,
+      fn: async () => {
         const token = `E2E_OK_${Date.now()}`
 
-        await llm.text(token)
-        await project.gotoSession()
+        await llm.textMatch(titleMatch, "E2E Title")
+        await llm.textMatch(promptMatch(token), token)
 
-        const prompt = page.locator(promptSelector)
-        await prompt.click()
-        await page.keyboard.type(`Reply with exactly: ${token}`)
-        await page.keyboard.press("Enter")
+        await withBackendProject(
+          async (project) => {
+            const prompt = page.locator(promptSelector)
+            await prompt.click()
+            await page.keyboard.type(`Reply with exactly: ${token}`)
+            await page.keyboard.press("Enter")
 
-        await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
+            await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
 
-        const sessionID = (() => {
-          const id = sessionIDFromUrl(page.url())
-          if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
-          return id
-        })()
-        project.trackSession(sessionID)
+            const sessionID = (() => {
+              const id = sessionIDFromUrl(page.url())
+              if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
+              return id
+            })()
+            project.trackSession(sessionID)
 
-        await expect
-          .poll(
-            async () => {
-              const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
-              return messages
-                .filter((m) => m.info.role === "assistant")
-                .flatMap((m) => m.parts)
-                .filter((p) => p.type === "text")
-                .map((p) => p.text)
-                .join("\n")
-            },
-            { timeout: 30_000 },
-          )
-          .toContain(token)
-      },
-      {
-        model: { providerID: "e2e-llm", modelID: "test-model" },
-        setup: (dir) => config(dir, llm.url),
+            await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
+
+            await expect
+              .poll(() => assistantText(project.sdk, sessionID), { timeout: 30_000 })
+              .toContain(token)
+          },
+          {
+            model: openaiModel,
+          },
+        )
       },
-    )
+    })
   } finally {
     page.off("pageerror", onPageError)
   }