test(app): add a golden path for mocked e2e prompts (#20593)

1 files changed, 424 insertions, 125 deletions

diff --git a/packages/app/e2e/fixtures.ts b/packages/app/e2e/fixtures.ts
index 8c018a9f0..fe2eb9c1a 100644
--- a/packages/app/e2e/fixtures.ts
+++ b/packages/app/e2e/fixtures.ts
@@ -10,13 +10,14 @@ import {
   cleanupTestProject,
   createTestProject,
   setHealthPhase,
-  seedProjects,
   sessionIDFromUrl,
-  waitSlug,
   waitSession,
+  waitSessionIdle,
+  waitSessionSaved,
+  waitSlug,
 } from "./actions"
-import { openaiModel, withMockOpenAI } from "./prompt/mock"
-import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"
+import { promptSelector } from "./selectors"
+import { createSdk, dirSlug, getWorktree, serverUrl, sessionPath } from "./utils"
 
 type LLMFixture = {
   url: string
@@ -51,6 +52,23 @@ type LLMFixture = {
   misses: () => Promise<Array<{ url: URL; body: Record<string, unknown> }>>
 }
 
+type LLMWorker = LLMFixture & {
+  reset: () => Promise<void>
+}
+
+type AssistantFixture = {
+  reply: LLMFixture["text"]
+  tool: LLMFixture["tool"]
+  toolHang: LLMFixture["toolHang"]
+  reason: LLMFixture["reason"]
+  fail: LLMFixture["fail"]
+  error: LLMFixture["error"]
+  hang: LLMFixture["hang"]
+  hold: LLMFixture["hold"]
+  calls: LLMFixture["calls"]
+  pending: LLMFixture["pending"]
+}
+
 export const settingsKey = "settings.v3"
 
 const seedModel = (() => {
@@ -63,6 +81,40 @@ const seedModel = (() => {
   }
 })()
 
+function clean(value: string | null) {
+  return (value ?? "").replace(/\u200B/g, "").trim()
+}
+
+async function visit(page: Page, url: string) {
+  let err: unknown
+  for (const _ of [0, 1, 2]) {
+    try {
+      await page.goto(url)
+      return
+    } catch (cause) {
+      err = cause
+      if (!String(cause).includes("ERR_CONNECTION_REFUSED")) throw cause
+      await new Promise((resolve) => setTimeout(resolve, 300))
+    }
+  }
+  throw err
+}
+
+async function promptSend(page: Page) {
+  return page
+    .evaluate(() => {
+      const win = window as E2EWindow
+      const sent = win.__opencode_e2e?.prompt?.sent
+      return {
+        started: sent?.started ?? 0,
+        count: sent?.count ?? 0,
+        sessionID: sent?.sessionID,
+        directory: sent?.directory,
+      }
+    })
+    .catch(() => ({ started: 0, count: 0, sessionID: undefined, directory: undefined }))
+}
+
 type ProjectHandle = {
   directory: string
   slug: string
@@ -79,16 +131,23 @@ type ProjectOptions = {
   beforeGoto?: (project: { directory: string; sdk: ReturnType<typeof createSdk> }) => Promise<void>
 }
 
+type ProjectFixture = ProjectHandle & {
+  open: (options?: ProjectOptions) => Promise<void>
+  prompt: (text: string) => Promise<string>
+  user: (text: string) => Promise<string>
+  shell: (cmd: string) => Promise<string>
+}
+
 type TestFixtures = {
   llm: LLMFixture
+  assistant: AssistantFixture
+  project: ProjectFixture
   sdk: ReturnType<typeof createSdk>
   gotoSession: (sessionID?: string) => Promise<void>
-  withProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
-  withBackendProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
-  withMockProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
 }
 
 type WorkerFixtures = {
+  _llm: LLMWorker
   backend: {
     url: string
     sdk: (directory?: string) => ReturnType<typeof createSdk>
@@ -98,9 +157,42 @@ type WorkerFixtures = {
 }
 
 export const test = base.extend<TestFixtures, WorkerFixtures>({
+  _llm: [
+    async ({}, use) => {
+      const rt = ManagedRuntime.make(TestLLMServer.layer)
+      try {
+        const svc = await rt.runPromise(TestLLMServer.asEffect())
+        await use({
+          url: svc.url,
+          push: (...input) => rt.runPromise(svc.push(...input)),
+          pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)),
+          textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)),
+          toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)),
+          text: (value, opts) => rt.runPromise(svc.text(value, opts)),
+          tool: (name, input) => rt.runPromise(svc.tool(name, input)),
+          toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),
+          reason: (value, opts) => rt.runPromise(svc.reason(value, opts)),
+          fail: (message) => rt.runPromise(svc.fail(message)),
+          error: (status, body) => rt.runPromise(svc.error(status, body)),
+          hang: () => rt.runPromise(svc.hang),
+          hold: (value, wait) => rt.runPromise(svc.hold(value, wait)),
+          reset: () => rt.runPromise(svc.reset),
+          hits: () => rt.runPromise(svc.hits),
+          calls: () => rt.runPromise(svc.calls),
+          wait: (count) => rt.runPromise(svc.wait(count)),
+          inputs: () => rt.runPromise(svc.inputs),
+          pending: () => rt.runPromise(svc.pending),
+          misses: () => rt.runPromise(svc.misses),
+        })
+      } finally {
+        await rt.dispose()
+      }
+    },
+    { scope: "worker" },
+  ],
   backend: [
-    async ({}, use, workerInfo) => {
-      const handle = await startBackend(`w${workerInfo.workerIndex}`)
+    async ({ _llm }, use, workerInfo) => {
+      const handle = await startBackend(`w${workerInfo.workerIndex}`, { llmUrl: _llm.url })
       try {
         await use({
           url: handle.url,
@@ -112,35 +204,48 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
     },
     { scope: "worker" },
   ],
-  llm: async ({}, use) => {
-    const rt = ManagedRuntime.make(TestLLMServer.layer)
-    try {
-      const svc = await rt.runPromise(TestLLMServer.asEffect())
-      await use({
-        url: svc.url,
-        push: (...input) => rt.runPromise(svc.push(...input)),
-        pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)),
-        textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)),
-        toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)),
-        text: (value, opts) => rt.runPromise(svc.text(value, opts)),
-        tool: (name, input) => rt.runPromise(svc.tool(name, input)),
-        toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),
-        reason: (value, opts) => rt.runPromise(svc.reason(value, opts)),
-        fail: (message) => rt.runPromise(svc.fail(message)),
-        error: (status, body) => rt.runPromise(svc.error(status, body)),
-        hang: () => rt.runPromise(svc.hang),
-        hold: (value, wait) => rt.runPromise(svc.hold(value, wait)),
-        hits: () => rt.runPromise(svc.hits),
-        calls: () => rt.runPromise(svc.calls),
-        wait: (count) => rt.runPromise(svc.wait(count)),
-        inputs: () => rt.runPromise(svc.inputs),
-        pending: () => rt.runPromise(svc.pending),
-        misses: () => rt.runPromise(svc.misses),
-      })
-    } finally {
-      await rt.dispose()
+  llm: async ({ _llm }, use) => {
+    await _llm.reset()
+    await use({
+      url: _llm.url,
+      push: _llm.push,
+      pushMatch: _llm.pushMatch,
+      textMatch: _llm.textMatch,
+      toolMatch: _llm.toolMatch,
+      text: _llm.text,
+      tool: _llm.tool,
+      toolHang: _llm.toolHang,
+      reason: _llm.reason,
+      fail: _llm.fail,
+      error: _llm.error,
+      hang: _llm.hang,
+      hold: _llm.hold,
+      hits: _llm.hits,
+      calls: _llm.calls,
+      wait: _llm.wait,
+      inputs: _llm.inputs,
+      pending: _llm.pending,
+      misses: _llm.misses,
+    })
+    const pending = await _llm.pending()
+    if (pending > 0) {
+      throw new Error(`TestLLMServer still has ${pending} queued response(s) after the test finished`)
     }
   },
+  assistant: async ({ llm }, use) => {
+    await use({
+      reply: llm.text,
+      tool: llm.tool,
+      toolHang: llm.toolHang,
+      reason: llm.reason,
+      fail: llm.fail,
+      error: llm.error,
+      hang: llm.hang,
+      hold: llm.hold,
+      calls: llm.calls,
+      pending: llm.pending,
+    })
+  },
   page: async ({ page }, use) => {
     let boundary: string | undefined
     setHealthPhase(page, "test")
@@ -165,9 +270,8 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
     if (boundary) throw new Error(boundary)
   },
   directory: [
-    async ({}, use) => {
-      const directory = await getWorktree()
-      await use(directory)
+    async ({ backend }, use) => {
+      await use(await getWorktree(backend.url))
     },
     { scope: "worker" },
   ],
@@ -177,93 +281,254 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
     },
     { scope: "worker" },
   ],
-  sdk: async ({ directory }, use) => {
-    await use(createSdk(directory))
+  sdk: async ({ directory, backend }, use) => {
+    await use(backend.sdk(directory))
   },
-  gotoSession: async ({ page, directory }, use) => {
-    await seedStorage(page, { directory })
+  gotoSession: async ({ page, directory, backend }, use) => {
+    await seedStorage(page, { directory, serverUrl: backend.url })
 
     const gotoSession = async (sessionID?: string) => {
-      await page.goto(sessionPath(directory, sessionID))
-      await waitSession(page, { directory, sessionID })
+      await visit(page, sessionPath(directory, sessionID))
+      await waitSession(page, {
+        directory,
+        sessionID,
+        serverUrl: backend.url,
+        allowAnySession: !sessionID,
+      })
     }
     await use(gotoSession)
   },
-  withProject: async ({ page }, use) => {
-    await use((callback, options) => runProject(page, callback, options))
-  },
-  withBackendProject: async ({ page, backend }, use) => {
-    await use((callback, options) =>
-      runProject(page, callback, { ...options, serverUrl: backend.url, sdk: backend.sdk }),
-    )
-  },
-  withMockProject: async ({ page, llm, backend }, use) => {
-    await use((callback, options) =>
-      withMockOpenAI({
-        serverUrl: backend.url,
-        llmUrl: llm.url,
-        fn: () =>
-          runProject(page, callback, {
-            ...options,
-            model: options?.model ?? openaiModel,
-            serverUrl: backend.url,
-            sdk: backend.sdk,
-          }),
-      }),
-    )
+  project: async ({ page, llm, backend }, use) => {
+    const item = makeProject(page, llm, backend)
+    try {
+      await use(item.project)
+    } finally {
+      await item.cleanup()
+    }
   },
 })
 
-async function runProject<T>(
+function makeProject(
   page: Page,
-  callback: (project: ProjectHandle) => Promise<T>,
-  options?: ProjectOptions & {
-    serverUrl?: string
-    sdk?: (directory?: string) => ReturnType<typeof createSdk>
-  },
+  llm: LLMFixture,
+  backend: { url: string; sdk: (directory?: string) => ReturnType<typeof createSdk> },
 ) {
-  const url = options?.serverUrl
-  const root = await createTestProject(url ? { serverUrl: url } : undefined)
-  const sdk = options?.sdk?.(root) ?? createSdk(root, url)
-  const sessions = new Map<string, string>()
-  const dirs = new Set<string>()
-  await options?.setup?.(root)
-  await seedStorage(page, {
-    directory: root,
-    extra: options?.extra,
-    model: options?.model,
-    serverUrl: url,
-  })
+  let state:
+    | {
+        directory: string
+        slug: string
+        sdk: ReturnType<typeof createSdk>
+        sessions: Map<string, string>
+        dirs: Set<string>
+      }
+    | undefined
 
-  const gotoSession = async (sessionID?: string) => {
-    await page.goto(sessionPath(root, sessionID))
-    await waitSession(page, { directory: root, sessionID, serverUrl: url })
-    const current = sessionIDFromUrl(page.url())
-    if (current) trackSession(current)
+  const need = () => {
+    if (state) return state
+    throw new Error("project.open() must be called first")
   }
 
   const trackSession = (sessionID: string, directory?: string) => {
-    sessions.set(sessionID, directory ?? root)
+    const cur = need()
+    cur.sessions.set(sessionID, directory ?? cur.directory)
   }
 
   const trackDirectory = (directory: string) => {
-    if (directory !== root) dirs.add(directory)
+    const cur = need()
+    if (directory !== cur.directory) cur.dirs.add(directory)
+  }
+
+  const gotoSession = async (sessionID?: string) => {
+    const cur = need()
+    await visit(page, sessionPath(cur.directory, sessionID))
+    await waitSession(page, {
+      directory: cur.directory,
+      sessionID,
+      serverUrl: backend.url,
+      allowAnySession: !sessionID,
+    })
+    const current = sessionIDFromUrl(page.url())
+    if (current) trackSession(current)
   }
 
-  try {
-    await options?.beforeGoto?.({ directory: root, sdk })
+  const open = async (options?: ProjectOptions) => {
+    if (state) return
+    const directory = await createTestProject({ serverUrl: backend.url })
+    const sdk = backend.sdk(directory)
+    await options?.setup?.(directory)
+    await seedStorage(page, {
+      directory,
+      extra: options?.extra,
+      model: options?.model,
+      serverUrl: backend.url,
+    })
+    state = {
+      directory,
+      slug: "",
+      sdk,
+      sessions: new Map(),
+      dirs: new Set(),
+    }
+    await options?.beforeGoto?.({ directory, sdk })
     await gotoSession()
-    const slug = await waitSlug(page)
-    return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory, sdk })
-  } finally {
+    need().slug = await waitSlug(page)
+  }
+
+  const send = async (text: string, input: { noReply: boolean; shell: boolean }) => {
+    if (input.noReply) {
+      const cur = need()
+      const state = await page.evaluate(() => {
+        const model = (window as E2EWindow).__opencode_e2e?.model?.current
+        if (!model) return null
+        return {
+          dir: model.dir,
+          sessionID: model.sessionID,
+          agent: model.agent,
+          model: model.model ? { providerID: model.model.providerID, modelID: model.model.modelID } : undefined,
+          variant: model.variant ?? undefined,
+        }
+      })
+      const dir = state?.dir ?? cur.directory
+      const sdk = backend.sdk(dir)
+      const sessionID = state?.sessionID
+        ? state.sessionID
+        : await sdk.session.create({ directory: dir, title: "E2E Session" }).then((res) => {
+            if (!res.data?.id) throw new Error("Failed to create no-reply session")
+            return res.data.id
+          })
+      await sdk.session.prompt({
+        sessionID,
+        agent: state?.agent,
+        model: state?.model,
+        variant: state?.variant,
+        noReply: true,
+        parts: [{ type: "text", text }],
+      })
+      await visit(page, sessionPath(dir, sessionID))
+      const active = await waitSession(page, {
+        directory: dir,
+        sessionID,
+        serverUrl: backend.url,
+      })
+      trackSession(sessionID, active.directory)
+      await waitSessionSaved(active.directory, sessionID, 90_000, backend.url)
+      return sessionID
+    }
+
+    const prev = await promptSend(page)
+    if (!input.noReply && !input.shell && (await llm.pending()) === 0) {
+      await llm.text("ok")
+    }
+
+    const prompt = page.locator(promptSelector).first()
+    const submit = async () => {
+      await expect(prompt).toBeVisible()
+      await prompt.click()
+      if (input.shell) {
+        await page.keyboard.type("!")
+        await expect(prompt).toHaveAttribute("aria-label", /enter shell command/i)
+      }
+      await page.keyboard.type(text)
+      await expect.poll(async () => clean(await prompt.textContent())).toBe(text)
+      await page.keyboard.press("Enter")
+      const started = await expect
+        .poll(async () => (await promptSend(page)).started, { timeout: 5_000 })
+        .toBeGreaterThan(prev.started)
+        .then(() => true)
+        .catch(() => false)
+      if (started) return
+      const send = page.getByRole("button", { name: "Send" }).first()
+      const enabled = await send
+        .isEnabled()
+        .then((x) => x)
+        .catch(() => false)
+      if (enabled) {
+        await send.click()
+      } else {
+        await prompt.click()
+        await page.keyboard.press("Enter")
+      }
+      await expect.poll(async () => (await promptSend(page)).started, { timeout: 5_000 }).toBeGreaterThan(prev.started)
+    }
+
+    await submit()
+
+    let next: { sessionID: string; directory: string } | undefined
+    await expect
+      .poll(
+        async () => {
+          const sent = await promptSend(page)
+          if (sent.count <= prev.count) return ""
+          if (!sent.sessionID || !sent.directory) return ""
+          next = { sessionID: sent.sessionID, directory: sent.directory }
+          return sent.sessionID
+        },
+        { timeout: 90_000 },
+      )
+      .not.toBe("")
+
+    if (!next) throw new Error("Failed to observe prompt submission in e2e prompt probe")
+    const active = await waitSession(page, {
+      directory: next.directory,
+      sessionID: next.sessionID,
+      serverUrl: backend.url,
+    })
+    trackSession(next.sessionID, active.directory)
+    if (!input.shell) {
+      await waitSessionSaved(active.directory, next.sessionID, 90_000, backend.url)
+    }
+    await waitSessionIdle(backend.sdk(active.directory), next.sessionID, 90_000).catch(() => undefined)
+    return next.sessionID
+  }
+
+  const prompt = async (text: string) => {
+    return send(text, { noReply: false, shell: false })
+  }
+
+  const user = async (text: string) => {
+    return send(text, { noReply: true, shell: false })
+  }
+
+  const shell = async (cmd: string) => {
+    return send(cmd, { noReply: false, shell: true })
+  }
+
+  const cleanup = async () => {
+    const cur = state
+    if (!cur) return
     setHealthPhase(page, "cleanup")
     await Promise.allSettled(
-      Array.from(sessions, ([sessionID, directory]) => cleanupSession({ sessionID, directory, serverUrl: url })),
+      Array.from(cur.sessions, ([sessionID, directory]) =>
+        cleanupSession({ sessionID, directory, serverUrl: backend.url }),
+      ),
     )
-    await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory)))
-    await cleanupTestProject(root)
+    await Promise.allSettled(Array.from(cur.dirs, (directory) => cleanupTestProject(directory)))
+    await cleanupTestProject(cur.directory)
+    state = undefined
     setHealthPhase(page, "test")
   }
+
+  return {
+    project: {
+      open,
+      prompt,
+      user,
+      shell,
+      gotoSession,
+      trackSession,
+      trackDirectory,
+      get directory() {
+        return need().directory
+      },
+      get slug() {
+        return need().slug
+      },
+      get sdk() {
+        return need().sdk
+      },
+    },
+    cleanup,
+  }
 }
 
 async function seedStorage(
@@ -275,31 +540,65 @@ async function seedStorage(
     serverUrl?: string
   },
 ) {
-  await seedProjects(page, input)
-  await page.addInitScript((model: { providerID: string; modelID: string }) => {
-    const win = window as E2EWindow
-    win.__opencode_e2e = {
-      ...win.__opencode_e2e,
-      model: {
-        enabled: true,
-      },
-      prompt: {
-        enabled: true,
-      },
-      terminal: {
-        enabled: true,
-        terminals: {},
-      },
-    }
-    localStorage.setItem(
-      "opencode.global.dat:model",
-      JSON.stringify({
-        recent: [model],
-        user: [],
-        variant: {},
-      }),
-    )
-  }, input.model ?? seedModel)
+  const origin = input.serverUrl ?? serverUrl
+  await page.addInitScript(
+    (args: {
+      directory: string
+      serverUrl: string
+      extra: string[]
+      model: { providerID: string; modelID: string }
+    }) => {
+      const key = "opencode.global.dat:server"
+      const raw = localStorage.getItem(key)
+      const parsed = (() => {
+        if (!raw) return undefined
+        try {
+          return JSON.parse(raw) as unknown
+        } catch {
+          return undefined
+        }
+      })()
+
+      const store = parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : {}
+      const list = Array.isArray(store.list) ? store.list : []
+      const lastProject = store.lastProject && typeof store.lastProject === "object" ? store.lastProject : {}
+      const projects = store.projects && typeof store.projects === "object" ? store.projects : {}
+      const next = { ...(projects as Record<string, unknown>) }
+      const nextList = list.includes(args.serverUrl) ? list : [args.serverUrl, ...list]
+
+      const add = (origin: string, directory: string) => {
+        const current = next[origin]
+        const items = Array.isArray(current) ? current : []
+        const existing = items.filter(
+          (p): p is { worktree: string; expanded?: boolean } =>
+            !!p &&
+            typeof p === "object" &&
+            "worktree" in p &&
+            typeof (p as { worktree?: unknown }).worktree === "string",
+        )
+        if (existing.some((p) => p.worktree === directory)) return
+        next[origin] = [{ worktree: directory, expanded: true }, ...existing]
+      }
+
+      for (const directory of [args.directory, ...args.extra]) {
+        add("local", directory)
+        add(args.serverUrl, directory)
+      }
+
+      localStorage.setItem(key, JSON.stringify({ list: nextList, projects: next, lastProject }))
+      localStorage.setItem("opencode.settings.dat:defaultServerUrl", args.serverUrl)
+
+      const win = window as E2EWindow
+      win.__opencode_e2e = {
+        ...win.__opencode_e2e,
+        model: { enabled: true },
+        prompt: { enabled: true },
+        terminal: { enabled: true, terminals: {} },
+      }
+      localStorage.setItem("opencode.global.dat:model", JSON.stringify({ recent: [args.model], user: [], variant: {} }))
+    },
+    { directory: input.directory, serverUrl: origin, extra: input.extra ?? [], model: input.model ?? seedModel },
+  )
 }
 
 export { expect }