test(app): block real llm calls in e2e prompts (#20579)

5 files changed, 67 insertions, 10 deletions

diff --git a/packages/app/e2e/actions.ts b/packages/app/e2e/actions.ts
index dc023ddc0..df8e0768e 100644
--- a/packages/app/e2e/actions.ts
+++ b/packages/app/e2e/actions.ts
@@ -1,5 +1,5 @@
 import { base64Decode, base64Encode } from "@opencode-ai/util/encode"
-import { expect, type Locator, type Page } from "@playwright/test"
+import { expect, type Locator, type Page, type Route } from "@playwright/test"
 import fs from "node:fs/promises"
 import os from "node:os"
 import path from "node:path"
@@ -43,6 +43,27 @@ export async function defocus(page: Page) {
     .catch(() => undefined)
 }
 
+export async function withNoReplyPrompt<T>(page: Page, fn: () => Promise<T>) {
+  const url = "**/session/*/prompt_async"
+  const route = async (input: Route) => {
+    const body = input.request().postDataJSON()
+    await input.continue({
+      postData: JSON.stringify({ ...body, noReply: true }),
+      headers: {
+        ...input.request().headers(),
+        "content-type": "application/json",
+      },
+    })
+  }
+
+  await page.route(url, route)
+  try {
+    return await fn()
+  } finally {
+    await page.unroute(url, route)
+  }
+}
+
 async function terminalID(term: Locator) {
   const id = await term.getAttribute(terminalAttr)
   if (id) return id
diff --git a/packages/app/e2e/projects/projects-switch.spec.ts b/packages/app/e2e/projects/projects-switch.spec.ts
index b46c1b407..f87a47cf0 100644
--- a/packages/app/e2e/projects/projects-switch.spec.ts
+++ b/packages/app/e2e/projects/projects-switch.spec.ts
@@ -10,6 +10,7 @@ import {
   waitSession,
   waitSessionSaved,
   waitSlug,
+  withNoReplyPrompt,
 } from "../actions"
 import { projectSwitchSelector, promptSelector, workspaceItemSelector, workspaceNewSessionSelector } from "../selectors"
 import { dirSlug, resolveDirectory } from "../utils"
@@ -81,8 +82,10 @@ test("switching back to a project opens the latest workspace session", async ({
         // Create a session by sending a prompt
         const prompt = page.locator(promptSelector)
         await expect(prompt).toBeVisible()
-        await prompt.fill("test")
-        await page.keyboard.press("Enter")
+        await withNoReplyPrompt(page, async () => {
+          await prompt.fill("test")
+          await page.keyboard.press("Enter")
+        })
 
         // Wait for the URL to update with the new session ID
         await expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 15_000 }).not.toBe("")
diff --git a/packages/app/e2e/projects/workspace-new-session.spec.ts b/packages/app/e2e/projects/workspace-new-session.spec.ts
index 3a7a6bbc2..835c8c99e 100644
--- a/packages/app/e2e/projects/workspace-new-session.spec.ts
+++ b/packages/app/e2e/projects/workspace-new-session.spec.ts
@@ -9,6 +9,7 @@ import {
   waitSession,
   waitSessionSaved,
   waitSlug,
+  withNoReplyPrompt,
 } from "../actions"
 import { promptSelector, workspaceItemSelector, workspaceNewSessionSelector } from "../selectors"
 import { createSdk } from "../utils"
@@ -58,8 +59,10 @@ async function createSessionFromWorkspace(
 
   const prompt = page.locator(promptSelector)
   await expect(prompt).toBeVisible()
-  await prompt.fill(text)
-  await page.keyboard.press("Enter")
+  await withNoReplyPrompt(page, async () => {
+    await prompt.fill(text)
+    await page.keyboard.press("Enter")
+  })
 
   await expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 15_000 }).not.toBe("")
   const sessionID = sessionIDFromUrl(page.url())
diff --git a/packages/app/e2e/session/session-model-persistence.spec.ts b/packages/app/e2e/session/session-model-persistence.spec.ts
index 36cbb0fbf..66bc451bc 100644
--- a/packages/app/e2e/session/session-model-persistence.spec.ts
+++ b/packages/app/e2e/session/session-model-persistence.spec.ts
@@ -8,11 +8,11 @@ import {
   waitSession,
   waitSessionIdle,
   waitSlug,
+  withNoReplyPrompt,
 } from "../actions"
 import {
   promptAgentSelector,
   promptModelSelector,
-  promptSelector,
   promptVariantSelector,
   workspaceItemSelector,
   workspaceNewSessionSelector,
@@ -231,11 +231,14 @@ async function goto(page: Page, directory: string, sessionID?: string) {
 }
 
 async function submit(page: Page, value: string) {
-  const prompt = page.locator(promptSelector)
+  const prompt = page.locator('[data-component="prompt-input"]')
   await expect(prompt).toBeVisible()
-  await prompt.click()
-  await prompt.fill(value)
-  await prompt.press("Enter")
+
+  await withNoReplyPrompt(page, async () => {
+    await prompt.click()
+    await prompt.fill(value)
+    await prompt.press("Enter")
+  })
 
   await expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 30_000 }).not.toBe("")
   const id = sessionIDFromUrl(page.url())
diff --git a/packages/app/test/e2e/no-real-llm.test.ts b/packages/app/test/e2e/no-real-llm.test.ts
new file mode 100644
index 000000000..c801df56f
--- /dev/null
+++ b/packages/app/test/e2e/no-real-llm.test.ts
@@ -0,0 +1,27 @@
+import { describe, expect, test } from "bun:test"
+import path from "node:path"
+import { fileURLToPath } from "node:url"
+
+const dir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../e2e")
+
+function hasPrompt(src: string) {
+  if (!src.includes("withProject(")) return false
+  if (src.includes("withNoReplyPrompt(")) return false
+  if (src.includes("session.promptAsync({") && !src.includes("noReply: true")) return true
+  if (!src.includes("promptSelector")) return false
+  return src.includes('keyboard.press("Enter")') || src.includes('prompt.press("Enter")')
+}
+
+describe("e2e llm guard", () => {
+  test("withProject specs do not submit prompt replies", async () => {
+    const bad: string[] = []
+
+    for await (const file of new Bun.Glob("**/*.spec.ts").scan({ cwd: dir, absolute: true })) {
+      const src = await Bun.file(file).text()
+      if (!hasPrompt(src)) continue
+      bad.push(path.relative(dir, file))
+    }
+
+    expect(bad).toEqual([])
+  })
+})