diff options
| author | Adam Malczewski <[email protected]> | 2026-06-21 13:11:29 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-21 13:11:29 +0900 |
| commit | 8a4a624d16422467a8e85434c674bb591877e8ea (patch) | |
| tree | 54052da00bbc580742913e5c031b7cc1b160db19 /packages/tool-web-search/src/client.ts | |
| parent | d23de3254374d4d63c8e15c6ab9311c3c6f4da5b (diff) | |
| download | dispatch-8a4a624d16422467a8e85434c674bb591877e8ea.tar.gz dispatch-8a4a624d16422467a8e85434c674bb591877e8ea.zip | |
feat(tool-web-search): Firecrawl-backed web search tool
New standard tool extension with one tool web_search supporting 4 modes
(search, scrape, crawl, map) against a self-hosted Firecrawl instance.
Pure core: validateArgs (discriminated union by mode) + format* functions
+ truncateOutput. Injected edge: FirecrawlClient (injectable fetchFn/sleep/now,
AbortSignal.any for per-request timeout + caller cancellation). concurrencySafe
true, capabilities network. 38 tests, zero vi.mock.
Live-verified: umans-glm-5.2 called web_search → real Firecrawl results (also
the first live Umans API call).
Diffstat (limited to 'packages/tool-web-search/src/client.ts')
| -rw-r--r-- | packages/tool-web-search/src/client.ts | 243 |
1 files changed, 243 insertions, 0 deletions
diff --git a/packages/tool-web-search/src/client.ts b/packages/tool-web-search/src/client.ts new file mode 100644 index 0000000..071ba97 --- /dev/null +++ b/packages/tool-web-search/src/client.ts @@ -0,0 +1,243 @@ +/** + * FirecrawlClient — the injected outermost edge for the web_search tool. + * + * All effects (fetch, sleep, clock) are injected so the pure decision logic + * remains testable without real I/O. The factory builds four methods + * (`search`, `scrape`, `crawl`, `map`) over a self-hosted Firecrawl instance + * (no API key). `crawl` polls a status URL until the crawl completes or fails. + */ + +import type { CrawlPage, ScrapeResult, SearchHit } from "./format.js"; + +export type FetchLike = typeof globalThis.fetch; + +export const DEFAULT_BASE_URL = "http://100.102.55.49:31329/v1"; +export const DEFAULT_TIMEOUT_MS = 30_000; +export const CRAWL_POLL_MS = 2_000; +export const CRAWL_MAX_WAIT_MS = 5 * 60 * 1_000; + +export interface SearchParams { + readonly query: string; + readonly limit: number; + readonly lang?: string; + readonly country?: string; + readonly scrapeOptions?: { + readonly formats: readonly string[]; + readonly onlyMainContent: boolean; + }; +} + +export interface ScrapeParams { + readonly url: string; + readonly formats: readonly string[]; +} + +export interface CrawlParams { + readonly url: string; + readonly limit: number; + readonly formats: readonly string[]; +} + +export interface FirecrawlClient { + readonly search: (params: SearchParams, signal: AbortSignal) => Promise<readonly SearchHit[]>; + readonly scrape: (params: ScrapeParams, signal: AbortSignal) => Promise<ScrapeResult>; + readonly crawl: (params: CrawlParams, signal: AbortSignal) => Promise<readonly CrawlPage[]>; + readonly map: (url: string, signal: AbortSignal) => Promise<readonly string[]>; +} + +export interface FirecrawlClientDeps { + readonly baseUrl: string; + readonly fetchFn: FetchLike; + readonly timeoutMs?: number; + readonly pollMs?: number; + readonly maxWaitMs?: number; + readonly now?: () => number; + readonly sleep?: (ms: number, signal: AbortSignal) => Promise<void>; +} + +interface SearchResponse { + readonly success: boolean; + readonly data?: readonly SearchHit[]; + readonly error?: string; +} + +interface ScrapeResponse { + readonly success: boolean; + readonly data?: { + readonly markdown?: string; + readonly metadata?: { readonly title?: string }; + }; + readonly error?: string; +} + +interface CrawlStartResponse { + readonly success: boolean; + readonly url?: string; + readonly error?: string; +} + +interface CrawlStatusResponse { + readonly status: string; + readonly data?: readonly CrawlPage[]; + readonly error?: string; +} + +interface MapResponse { + readonly success: boolean; + readonly links?: readonly string[]; + readonly error?: string; +} + +/** Default sleep: resolve after `ms`, reject on abort. */ +async function defaultSleep(ms: number, signal: AbortSignal): Promise<void> { + return new Promise<void>((resolve, reject) => { + if (signal.aborted) { + reject(new Error("Request aborted.")); + return; + } + let timer: ReturnType<typeof setTimeout> | undefined; + const onAbort = (): void => { + if (timer !== undefined) { + clearTimeout(timer); + } + reject(new Error("Request aborted.")); + }; + timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, ms); + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +/** + * Create a FirecrawlClient. Each method builds a fetch request, calls the + * injected `fetchFn`, and handles HTTP + JSON errors. The per-request timeout + * is combined with the caller's cancellation signal via `AbortSignal.any`. + */ +export function createFirecrawlClient(deps: FirecrawlClientDeps): FirecrawlClient { + const baseUrl = deps.baseUrl; + const fetchFn = deps.fetchFn; + const timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const pollMs = deps.pollMs ?? CRAWL_POLL_MS; + const maxWaitMs = deps.maxWaitMs ?? CRAWL_MAX_WAIT_MS; + const now = deps.now ?? Date.now; + const sleep = deps.sleep ?? defaultSleep; + + async function request( + method: "POST" | "GET", + url: string, + body: unknown, + signal: AbortSignal, + ): Promise<unknown> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const combined = AbortSignal.any([signal, controller.signal]); + try { + let response: Response; + try { + response = await fetchFn(url, { + method, + headers: + body !== undefined + ? { "Content-Type": "application/json", Accept: "application/json" } + : { Accept: "application/json" }, + body: body !== undefined ? JSON.stringify(body) : undefined, + signal: combined, + }); + } catch (err) { + if (signal.aborted) { + throw new Error("Request aborted."); + } + if (controller.signal.aborted) { + throw new Error(`Firecrawl request timed out after ${timeoutMs / 1000} seconds.`); + } + throw err; + } + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`HTTP ${response.status} ${response.statusText}${text ? `: ${text}` : ""}`); + } + try { + return await response.json(); + } catch { + throw new Error("Failed to parse Firecrawl response as JSON"); + } + } finally { + clearTimeout(timeout); + } + } + + async function post(endpoint: string, body: unknown, signal: AbortSignal): Promise<unknown> { + return request("POST", `${baseUrl}/${endpoint}`, body, signal); + } + + return { + async search(params: SearchParams, signal: AbortSignal): Promise<readonly SearchHit[]> { + const body: Record<string, unknown> = { query: params.query, limit: params.limit }; + if (params.lang !== undefined) { + body.lang = params.lang; + } + if (params.country !== undefined) { + body.country = params.country; + } + if (params.scrapeOptions !== undefined) { + body.scrapeOptions = params.scrapeOptions; + } + const json = (await post("search", body, signal)) as SearchResponse; + if (!json.success) { + throw new Error(json.error ?? "Unknown error"); + } + return json.data ?? []; + }, + + async scrape(params: ScrapeParams, signal: AbortSignal): Promise<ScrapeResult> { + const body = { + url: params.url, + formats: params.formats, + onlyMainContent: true, + }; + const json = (await post("scrape", body, signal)) as ScrapeResponse; + if (!json.success) { + throw new Error(json.error ?? "Unknown error"); + } + return json; + }, + + async crawl(params: CrawlParams, signal: AbortSignal): Promise<readonly CrawlPage[]> { + const body = { + url: params.url, + limit: params.limit, + scrapeOptions: { formats: params.formats, onlyMainContent: true }, + }; + const startJson = (await post("crawl", body, signal)) as CrawlStartResponse; + if (!startJson.success) { + throw new Error(startJson.error ?? "Unknown error"); + } + const statusUrl = startJson.url; + if (statusUrl === undefined) { + throw new Error("crawl response missing status URL."); + } + const started = now(); + while (now() - started < maxWaitMs) { + await sleep(pollMs, signal); + const status = (await request("GET", statusUrl, undefined, signal)) as CrawlStatusResponse; + if (status.status === "completed") { + return status.data ?? []; + } + if (status.status === "failed") { + throw new Error(`crawl failed: ${status.error ?? "unknown"}`); + } + } + throw new Error("crawl timed out waiting for completion."); + }, + + async map(url: string, signal: AbortSignal): Promise<readonly string[]> { + const json = (await post("map", { url }, signal)) as MapResponse; + if (!json.success) { + throw new Error(json.error ?? "Unknown error"); + } + return json.links ?? []; + }, + }; +} |
