diff options
| author | Dax <[email protected]> | 2026-04-25 10:59:17 -0400 |
|---|---|---|
| committer | GitHub <[email protected]> | 2026-04-25 10:59:17 -0400 |
| commit | 62ef2a220723a6d6cb050e523fcdfaa974dafdda (patch) | |
| tree | 214b03d016e18e4d8fe1bfc7209c1edd86547bbd /packages/core/src | |
| parent | 37aa8442dc023fad250f2573c8235a544789900c (diff) | |
| download | opencode-62ef2a220723a6d6cb050e523fcdfaa974dafdda.tar.gz opencode-62ef2a220723a6d6cb050e523fcdfaa974dafdda.zip | |
refactor: rename shared package to core (#24309)
Diffstat (limited to 'packages/core/src')
| -rw-r--r-- | packages/core/src/filesystem.ts | 236 | ||||
| -rw-r--r-- | packages/core/src/global.ts | 42 | ||||
| -rw-r--r-- | packages/core/src/types.d.ts | 46 | ||||
| -rw-r--r-- | packages/core/src/util/array.ts | 10 | ||||
| -rw-r--r-- | packages/core/src/util/binary.ts | 41 | ||||
| -rw-r--r-- | packages/core/src/util/effect-flock.ts | 283 | ||||
| -rw-r--r-- | packages/core/src/util/encode.ts | 51 | ||||
| -rw-r--r-- | packages/core/src/util/error.ts | 60 | ||||
| -rw-r--r-- | packages/core/src/util/flock.ts | 358 | ||||
| -rw-r--r-- | packages/core/src/util/fn.ts | 11 | ||||
| -rw-r--r-- | packages/core/src/util/glob.ts | 34 | ||||
| -rw-r--r-- | packages/core/src/util/hash.ts | 7 | ||||
| -rw-r--r-- | packages/core/src/util/identifier.ts | 48 | ||||
| -rw-r--r-- | packages/core/src/util/iife.ts | 3 | ||||
| -rw-r--r-- | packages/core/src/util/lazy.ts | 11 | ||||
| -rw-r--r-- | packages/core/src/util/module.ts | 10 | ||||
| -rw-r--r-- | packages/core/src/util/path.ts | 37 | ||||
| -rw-r--r-- | packages/core/src/util/retry.ts | 42 | ||||
| -rw-r--r-- | packages/core/src/util/slug.ts | 74 |
19 files changed, 1404 insertions, 0 deletions
diff --git a/packages/core/src/filesystem.ts b/packages/core/src/filesystem.ts new file mode 100644 index 000000000..44346be8f --- /dev/null +++ b/packages/core/src/filesystem.ts @@ -0,0 +1,236 @@ +import { NodeFileSystem } from "@effect/platform-node" +import { dirname, join, relative, resolve as pathResolve } from "path" +import { realpathSync } from "fs" +import * as NFS from "fs/promises" +import { lookup } from "mime-types" +import { Effect, FileSystem, Layer, Schema, Context } from "effect" +import type { PlatformError } from "effect/PlatformError" +import { Glob } from "./util/glob" + +export namespace AppFileSystem { + export class FileSystemError extends Schema.TaggedErrorClass<FileSystemError>()("FileSystemError", { + method: Schema.String, + cause: Schema.optional(Schema.Defect), + }) {} + + export type Error = PlatformError | FileSystemError + + export interface DirEntry { + readonly name: string + readonly type: "file" | "directory" | "symlink" | "other" + } + + export interface Interface extends FileSystem.FileSystem { + readonly isDir: (path: string) => Effect.Effect<boolean> + readonly isFile: (path: string) => Effect.Effect<boolean> + readonly existsSafe: (path: string) => Effect.Effect<boolean> + readonly readJson: (path: string) => Effect.Effect<unknown, Error> + readonly writeJson: (path: string, data: unknown, mode?: number) => Effect.Effect<void, Error> + readonly ensureDir: (path: string) => Effect.Effect<void, Error> + readonly writeWithDirs: (path: string, content: string | Uint8Array, mode?: number) => Effect.Effect<void, Error> + readonly readDirectoryEntries: (path: string) => Effect.Effect<DirEntry[], Error> + readonly findUp: (target: string, start: string, stop?: string) => Effect.Effect<string[], Error> + readonly up: (options: { targets: string[]; start: string; stop?: string }) => Effect.Effect<string[], Error> + readonly globUp: (pattern: string, start: string, stop?: string) => Effect.Effect<string[], Error> + readonly glob: (pattern: string, options?: Glob.Options) => Effect.Effect<string[], Error> + readonly globMatch: (pattern: string, filepath: string) => boolean + } + + export class Service extends Context.Service<Service, Interface>()("@opencode/FileSystem") {} + + export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem + + const existsSafe = Effect.fn("FileSystem.existsSafe")(function* (path: string) { + return yield* fs.exists(path).pipe(Effect.orElseSucceed(() => false)) + }) + + const isDir = Effect.fn("FileSystem.isDir")(function* (path: string) { + const info = yield* fs.stat(path).pipe(Effect.catch(() => Effect.void)) + return info?.type === "Directory" + }) + + const isFile = Effect.fn("FileSystem.isFile")(function* (path: string) { + const info = yield* fs.stat(path).pipe(Effect.catch(() => Effect.void)) + return info?.type === "File" + }) + + const readDirectoryEntries = Effect.fn("FileSystem.readDirectoryEntries")(function* (dirPath: string) { + return yield* Effect.tryPromise({ + try: async () => { + const entries = await NFS.readdir(dirPath, { withFileTypes: true }) + return entries.map( + (e): DirEntry => ({ + name: e.name, + type: e.isDirectory() ? "directory" : e.isSymbolicLink() ? "symlink" : e.isFile() ? "file" : "other", + }), + ) + }, + catch: (cause) => new FileSystemError({ method: "readDirectoryEntries", cause }), + }) + }) + + const readJson = Effect.fn("FileSystem.readJson")(function* (path: string) { + const text = yield* fs.readFileString(path) + return JSON.parse(text) + }) + + const writeJson = Effect.fn("FileSystem.writeJson")(function* (path: string, data: unknown, mode?: number) { + const content = JSON.stringify(data, null, 2) + yield* fs.writeFileString(path, content) + if (mode) yield* fs.chmod(path, mode) + }) + + const ensureDir = Effect.fn("FileSystem.ensureDir")(function* (path: string) { + yield* fs.makeDirectory(path, { recursive: true }) + }) + + const writeWithDirs = Effect.fn("FileSystem.writeWithDirs")(function* ( + path: string, + content: string | Uint8Array, + mode?: number, + ) { + const write = typeof content === "string" ? fs.writeFileString(path, content) : fs.writeFile(path, content) + + yield* write.pipe( + Effect.catchIf( + (e) => e.reason._tag === "NotFound", + () => + Effect.gen(function* () { + yield* fs.makeDirectory(dirname(path), { recursive: true }) + yield* write + }), + ), + ) + if (mode) yield* fs.chmod(path, mode) + }) + + const glob = Effect.fn("FileSystem.glob")(function* (pattern: string, options?: Glob.Options) { + return yield* Effect.tryPromise({ + try: () => Glob.scan(pattern, options), + catch: (cause) => new FileSystemError({ method: "glob", cause }), + }) + }) + + const findUp = Effect.fn("FileSystem.findUp")(function* (target: string, start: string, stop?: string) { + const result: string[] = [] + let current = start + while (true) { + const search = join(current, target) + if (yield* fs.exists(search)) result.push(search) + if (stop === current) break + const parent = dirname(current) + if (parent === current) break + current = parent + } + return result + }) + + const up = Effect.fn("FileSystem.up")(function* (options: { targets: string[]; start: string; stop?: string }) { + const result: string[] = [] + let current = options.start + while (true) { + for (const target of options.targets) { + const search = join(current, target) + if (yield* fs.exists(search)) result.push(search) + } + if (options.stop === current) break + const parent = dirname(current) + if (parent === current) break + current = parent + } + return result + }) + + const globUp = Effect.fn("FileSystem.globUp")(function* (pattern: string, start: string, stop?: string) { + const result: string[] = [] + let current = start + while (true) { + const matches = yield* glob(pattern, { cwd: current, absolute: true, include: "file", dot: true }).pipe( + Effect.catch(() => Effect.succeed([] as string[])), + ) + result.push(...matches) + if (stop === current) break + const parent = dirname(current) + if (parent === current) break + current = parent + } + return result + }) + + return Service.of({ + ...fs, + existsSafe, + isDir, + isFile, + readDirectoryEntries, + readJson, + writeJson, + ensureDir, + writeWithDirs, + findUp, + up, + globUp, + glob, + globMatch: Glob.match, + }) + }), + ) + + export const defaultLayer = layer.pipe(Layer.provide(NodeFileSystem.layer)) + + // Pure helpers that don't need Effect (path manipulation, sync operations) + export function mimeType(p: string): string { + return lookup(p) || "application/octet-stream" + } + + export function normalizePath(p: string): string { + if (process.platform !== "win32") return p + const resolved = pathResolve(windowsPath(p)) + try { + return realpathSync.native(resolved) + } catch { + return resolved + } + } + + export function normalizePathPattern(p: string): string { + if (process.platform !== "win32") return p + if (p === "*") return p + const match = p.match(/^(.*)[\\/]\*$/) + if (!match) return normalizePath(p) + const dir = /^[A-Za-z]:$/.test(match[1]) ? match[1] + "\\" : match[1] + return join(normalizePath(dir), "*") + } + + export function resolve(p: string): string { + const resolved = pathResolve(windowsPath(p)) + try { + return normalizePath(realpathSync(resolved)) + } catch (e: any) { + if (e?.code === "ENOENT") return normalizePath(resolved) + throw e + } + } + + export function windowsPath(p: string): string { + if (process.platform !== "win32") return p + return p + .replace(/^\/([a-zA-Z]):(?:[\\/]|$)/, (_, drive) => `${drive.toUpperCase()}:/`) + .replace(/^\/([a-zA-Z])(?:\/|$)/, (_, drive) => `${drive.toUpperCase()}:/`) + .replace(/^\/cygdrive\/([a-zA-Z])(?:\/|$)/, (_, drive) => `${drive.toUpperCase()}:/`) + .replace(/^\/mnt\/([a-zA-Z])(?:\/|$)/, (_, drive) => `${drive.toUpperCase()}:/`) + } + + export function overlaps(a: string, b: string) { + const relA = relative(a, b) + const relB = relative(b, a) + return !relA || !relA.startsWith("..") || !relB || !relB.startsWith("..") + } + + export function contains(parent: string, child: string) { + return !relative(parent, child).startsWith("..") + } +} diff --git a/packages/core/src/global.ts b/packages/core/src/global.ts new file mode 100644 index 000000000..538cc091b --- /dev/null +++ b/packages/core/src/global.ts @@ -0,0 +1,42 @@ +import path from "path" +import { xdgData, xdgCache, xdgConfig, xdgState } from "xdg-basedir" +import os from "os" +import { Context, Effect, Layer } from "effect" + +export namespace Global { + export class Service extends Context.Service<Service, Interface>()("@opencode/Global") {} + + export interface Interface { + readonly home: string + readonly data: string + readonly cache: string + readonly config: string + readonly state: string + readonly bin: string + readonly log: string + } + + export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const app = "opencode" + const home = process.env.OPENCODE_TEST_HOME ?? os.homedir() + const data = path.join(xdgData!, app) + const cache = path.join(xdgCache!, app) + const cfg = path.join(xdgConfig!, app) + const state = path.join(xdgState!, app) + const bin = path.join(cache, "bin") + const log = path.join(data, "log") + + return Service.of({ + home, + data, + cache, + config: cfg, + state, + bin, + log, + }) + }), + ) +} diff --git a/packages/core/src/types.d.ts b/packages/core/src/types.d.ts new file mode 100644 index 000000000..60e1639ad --- /dev/null +++ b/packages/core/src/types.d.ts @@ -0,0 +1,46 @@ +declare module "@npmcli/arborist" { + export interface ArboristOptions { + path: string + binLinks?: boolean + progress?: boolean + savePrefix?: string + ignoreScripts?: boolean + [key: string]: unknown + } + + export interface ArboristNode { + name: string + path: string + } + + export interface ArboristEdge { + to?: ArboristNode + } + + export interface ArboristTree { + edgesOut: Map<string, ArboristEdge> + } + + export interface ReifyOptions { + add?: string[] + save?: boolean + saveType?: "prod" | "dev" | "optional" | "peer" + [key: string]: unknown + } + + export class Arborist { + constructor(options: ArboristOptions) + loadVirtual(): Promise<ArboristTree | undefined> + reify(options?: ReifyOptions): Promise<ArboristTree> + } +} + +declare var Bun: + | { + file(path: string): { + text(): Promise<string> + json(): Promise<unknown> + } + write(path: string, content: string | Uint8Array): Promise<void> + } + | undefined diff --git a/packages/core/src/util/array.ts b/packages/core/src/util/array.ts new file mode 100644 index 000000000..1fb8ac69e --- /dev/null +++ b/packages/core/src/util/array.ts @@ -0,0 +1,10 @@ +export function findLast<T>( + items: readonly T[], + predicate: (item: T, index: number, items: readonly T[]) => boolean, +): T | undefined { + for (let i = items.length - 1; i >= 0; i -= 1) { + const item = items[i] + if (predicate(item, i, items)) return item + } + return undefined +} diff --git a/packages/core/src/util/binary.ts b/packages/core/src/util/binary.ts new file mode 100644 index 000000000..3d8f61851 --- /dev/null +++ b/packages/core/src/util/binary.ts @@ -0,0 +1,41 @@ +export namespace Binary { + export function search<T>(array: T[], id: string, compare: (item: T) => string): { found: boolean; index: number } { + let left = 0 + let right = array.length - 1 + + while (left <= right) { + const mid = Math.floor((left + right) / 2) + const midId = compare(array[mid]) + + if (midId === id) { + return { found: true, index: mid } + } else if (midId < id) { + left = mid + 1 + } else { + right = mid - 1 + } + } + + return { found: false, index: left } + } + + export function insert<T>(array: T[], item: T, compare: (item: T) => string): T[] { + const id = compare(item) + let left = 0 + let right = array.length + + while (left < right) { + const mid = Math.floor((left + right) / 2) + const midId = compare(array[mid]) + + if (midId < id) { + left = mid + 1 + } else { + right = mid + } + } + + array.splice(left, 0, item) + return array + } +} diff --git a/packages/core/src/util/effect-flock.ts b/packages/core/src/util/effect-flock.ts new file mode 100644 index 000000000..16bcf091b --- /dev/null +++ b/packages/core/src/util/effect-flock.ts @@ -0,0 +1,283 @@ +import path from "path" +import os from "os" +import { randomUUID } from "crypto" +import { Context, Effect, Function, Layer, Option, Schedule, Schema } from "effect" +import type { FileSystem, Scope } from "effect" +import type { PlatformError } from "effect/PlatformError" +import { AppFileSystem } from "../filesystem" +import { Global } from "../global" +import { Hash } from "./hash" + +export namespace EffectFlock { + // --------------------------------------------------------------------------- + // Errors + // --------------------------------------------------------------------------- + + export class LockTimeoutError extends Schema.TaggedErrorClass<LockTimeoutError>()("LockTimeoutError", { + key: Schema.String, + }) {} + + export class LockCompromisedError extends Schema.TaggedErrorClass<LockCompromisedError>()("LockCompromisedError", { + detail: Schema.String, + }) {} + + class ReleaseError extends Schema.TaggedErrorClass<ReleaseError>()("ReleaseError", { + detail: Schema.String, + cause: Schema.optional(Schema.Defect), + }) { + override get message() { + return this.detail + } + } + + /** Internal: signals "lock is held, retry later". Never leaks to callers. */ + class NotAcquired extends Schema.TaggedErrorClass<NotAcquired>()("NotAcquired", {}) {} + + export type LockError = LockTimeoutError | LockCompromisedError + + // --------------------------------------------------------------------------- + // Timing (baked in — no caller ever overrides these) + // --------------------------------------------------------------------------- + + const STALE_MS = 60_000 + const TIMEOUT_MS = 5 * 60_000 + const BASE_DELAY_MS = 100 + const MAX_DELAY_MS = 2_000 + const HEARTBEAT_MS = Math.max(100, Math.floor(STALE_MS / 3)) + + const retrySchedule = Schedule.exponential(BASE_DELAY_MS, 1.7).pipe( + Schedule.either(Schedule.spaced(MAX_DELAY_MS)), + Schedule.jittered, + Schedule.while((meta) => meta.elapsed < TIMEOUT_MS), + ) + + // --------------------------------------------------------------------------- + // Lock metadata schema + // --------------------------------------------------------------------------- + + const LockMetaJson = Schema.fromJsonString( + Schema.Struct({ + token: Schema.String, + pid: Schema.Number, + hostname: Schema.String, + createdAt: Schema.String, + }), + ) + + const decodeMeta = Schema.decodeUnknownSync(LockMetaJson) + const encodeMeta = Schema.encodeSync(LockMetaJson) + + // --------------------------------------------------------------------------- + // Service + // --------------------------------------------------------------------------- + + export interface Interface { + readonly acquire: (key: string, dir?: string) => Effect.Effect<void, LockError, Scope.Scope> + readonly withLock: { + (key: string, dir?: string): <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E | LockError, R> + <A, E, R>(body: Effect.Effect<A, E, R>, key: string, dir?: string): Effect.Effect<A, E | LockError, R> + } + } + + export class Service extends Context.Service<Service, Interface>()("EffectFlock") {} + + // --------------------------------------------------------------------------- + // Layer + // --------------------------------------------------------------------------- + + function wall() { + return performance.timeOrigin + performance.now() + } + + const mtimeMs = (info: FileSystem.File.Info) => Option.getOrElse(info.mtime, () => new Date(0)).getTime() + + const isPathGone = (e: PlatformError) => e.reason._tag === "NotFound" || e.reason._tag === "Unknown" + + export const layer: Layer.Layer<Service, never, Global.Service | AppFileSystem.Service> = Layer.effect( + Service, + Effect.gen(function* () { + const global = yield* Global.Service + const fs = yield* AppFileSystem.Service + const lockRoot = path.join(global.state, "locks") + const hostname = os.hostname() + const ensuredDirs = new Set<string>() + + // -- helpers (close over fs) -- + + const safeStat = (file: string) => + fs.stat(file).pipe( + Effect.catchIf(isPathGone, () => Effect.void), + Effect.orDie, + ) + + const forceRemove = (target: string) => fs.remove(target, { recursive: true }).pipe(Effect.ignore) + + /** Atomic mkdir — returns true if created, false if already exists, dies on other errors. */ + const atomicMkdir = (dir: string) => + fs.makeDirectory(dir, { mode: 0o700 }).pipe( + Effect.as(true), + Effect.catchIf( + (e) => e.reason._tag === "AlreadyExists", + () => Effect.succeed(false), + ), + Effect.orDie, + ) + + /** Write with exclusive create — compromised error if file already exists. */ + const exclusiveWrite = (filePath: string, content: string, lockDir: string, detail: string) => + fs.writeFileString(filePath, content, { flag: "wx" }).pipe( + Effect.catch(() => + Effect.gen(function* () { + yield* forceRemove(lockDir) + return yield* new LockCompromisedError({ detail }) + }), + ), + ) + + const cleanStaleBreaker = Effect.fnUntraced(function* (breakerPath: string) { + const bs = yield* safeStat(breakerPath) + if (bs && wall() - mtimeMs(bs) > STALE_MS) yield* forceRemove(breakerPath) + return false + }) + + const ensureDir = Effect.fnUntraced(function* (dir: string) { + if (ensuredDirs.has(dir)) return + yield* fs.makeDirectory(dir, { recursive: true }).pipe(Effect.orDie) + ensuredDirs.add(dir) + }) + + const isStale = Effect.fnUntraced(function* (lockDir: string, heartbeatPath: string, metaPath: string) { + const now = wall() + + const hb = yield* safeStat(heartbeatPath) + if (hb) return now - mtimeMs(hb) > STALE_MS + + const meta = yield* safeStat(metaPath) + if (meta) return now - mtimeMs(meta) > STALE_MS + + const dir = yield* safeStat(lockDir) + if (!dir) return false + + return now - mtimeMs(dir) > STALE_MS + }) + + // -- single lock attempt -- + + type Handle = { token: string; metaPath: string; heartbeatPath: string; lockDir: string } + + const tryAcquireLockDir = (lockDir: string, key: string) => + Effect.gen(function* () { + const token = randomUUID() + const metaPath = path.join(lockDir, "meta.json") + const heartbeatPath = path.join(lockDir, "heartbeat") + + // Atomic mkdir — the POSIX lock primitive + const created = yield* atomicMkdir(lockDir) + + if (!created) { + if (!(yield* isStale(lockDir, heartbeatPath, metaPath))) return yield* new NotAcquired() + + // Stale — race for breaker ownership + const breakerPath = lockDir + ".breaker" + + const claimed = yield* fs.makeDirectory(breakerPath, { mode: 0o700 }).pipe( + Effect.as(true), + Effect.catchIf( + (e) => e.reason._tag === "AlreadyExists", + () => cleanStaleBreaker(breakerPath), + ), + Effect.catchIf(isPathGone, () => Effect.succeed(false)), + Effect.orDie, + ) + + if (!claimed) return yield* new NotAcquired() + + // We own the breaker — double-check staleness, nuke, recreate + const recreated = yield* Effect.gen(function* () { + if (!(yield* isStale(lockDir, heartbeatPath, metaPath))) return false + yield* forceRemove(lockDir) + return yield* atomicMkdir(lockDir) + }).pipe(Effect.ensuring(forceRemove(breakerPath))) + + if (!recreated) return yield* new NotAcquired() + } + + // We own the lock dir — write heartbeat + meta with exclusive create + yield* exclusiveWrite(heartbeatPath, "", lockDir, "heartbeat already existed") + + const metaJson = encodeMeta({ token, pid: process.pid, hostname, createdAt: new Date().toISOString() }) + yield* exclusiveWrite(metaPath, metaJson, lockDir, "meta.json already existed") + + return { token, metaPath, heartbeatPath, lockDir } satisfies Handle + }).pipe( + Effect.withSpan("EffectFlock.tryAcquire", { + attributes: { key }, + }), + ) + + // -- retry wrapper (preserves Handle type) -- + + const acquireHandle = (lockfile: string, key: string): Effect.Effect<Handle, LockError> => + tryAcquireLockDir(lockfile, key).pipe( + Effect.retry({ + while: (err) => err._tag === "NotAcquired", + schedule: retrySchedule, + }), + Effect.catchTag("NotAcquired", () => Effect.fail(new LockTimeoutError({ key }))), + ) + + // -- release -- + + const release = (handle: Handle) => + Effect.gen(function* () { + const raw = yield* fs.readFileString(handle.metaPath).pipe( + Effect.catch((err) => { + if (isPathGone(err)) return Effect.die(new ReleaseError({ detail: "metadata missing" })) + return Effect.die(err) + }), + ) + + const parsed = yield* Effect.try({ + try: () => decodeMeta(raw), + catch: (cause) => new ReleaseError({ detail: "metadata invalid", cause }), + }).pipe(Effect.orDie) + + if (parsed.token !== handle.token) return yield* Effect.die(new ReleaseError({ detail: "token mismatch" })) + + yield* forceRemove(handle.lockDir) + }) + + // -- build service -- + + const acquire = Effect.fn("EffectFlock.acquire")(function* (key: string, dir?: string) { + const lockDir = dir ?? lockRoot + yield* ensureDir(lockDir) + + const lockfile = path.join(lockDir, Hash.fast(key) + ".lock") + + // acquireRelease: acquire is uninterruptible, release is guaranteed + const handle = yield* Effect.acquireRelease(acquireHandle(lockfile, key), (handle) => release(handle)) + + // Heartbeat fiber — scoped, so it's interrupted before release runs + yield* fs + .utimes(handle.heartbeatPath, new Date(), new Date()) + .pipe(Effect.ignore, Effect.repeat(Schedule.spaced(HEARTBEAT_MS)), Effect.forkScoped) + }) + + const withLock: Interface["withLock"] = Function.dual( + (args) => Effect.isEffect(args[0]), + <A, E, R>(body: Effect.Effect<A, E, R>, key: string, dir?: string): Effect.Effect<A, E | LockError, R> => + Effect.scoped( + Effect.gen(function* () { + yield* acquire(key, dir) + return yield* body + }), + ), + ) + + return Service.of({ acquire, withLock }) + }), + ) + + export const defaultLayer = layer.pipe(Layer.provide(AppFileSystem.defaultLayer), Layer.provide(Global.layer)) +} diff --git a/packages/core/src/util/encode.ts b/packages/core/src/util/encode.ts new file mode 100644 index 000000000..e4c6e70ac --- /dev/null +++ b/packages/core/src/util/encode.ts @@ -0,0 +1,51 @@ +export function base64Encode(value: string) { + const bytes = new TextEncoder().encode(value) + const binary = Array.from(bytes, (b) => String.fromCharCode(b)).join("") + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "") +} + +export function base64Decode(value: string) { + const binary = atob(value.replace(/-/g, "+").replace(/_/g, "/")) + const bytes = Uint8Array.from(binary, (c) => c.charCodeAt(0)) + return new TextDecoder().decode(bytes) +} + +export async function hash(content: string, algorithm = "SHA-256"): Promise<string> { + const encoder = new TextEncoder() + const data = encoder.encode(content) + const hashBuffer = await crypto.subtle.digest(algorithm, data) + const hashArray = Array.from(new Uint8Array(hashBuffer)) + const hashHex = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("") + return hashHex +} + +export function checksum(content: string): string | undefined { + if (!content) return undefined + let hash = 0x811c9dc5 + for (let i = 0; i < content.length; i++) { + hash ^= content.charCodeAt(i) + hash = Math.imul(hash, 0x01000193) + } + return (hash >>> 0).toString(36) +} + +export function sampledChecksum(content: string, limit = 500_000): string | undefined { + if (!content) return undefined + if (content.length <= limit) return checksum(content) + + const size = 4096 + const points = [ + 0, + Math.floor(content.length * 0.25), + Math.floor(content.length * 0.5), + Math.floor(content.length * 0.75), + content.length - size, + ] + const hashes = points + .map((point) => { + const start = Math.max(0, Math.min(content.length - size, point - Math.floor(size / 2))) + return checksum(content.slice(start, start + size)) ?? "" + }) + .join(":") + return `${content.length}:${hashes}` +} diff --git a/packages/core/src/util/error.ts b/packages/core/src/util/error.ts new file mode 100644 index 000000000..9d3b7c661 --- /dev/null +++ b/packages/core/src/util/error.ts @@ -0,0 +1,60 @@ +import z from "zod" + +export abstract class NamedError extends Error { + abstract schema(): z.core.$ZodType + abstract toObject(): { name: string; data: any } + + static hasName(error: unknown, name: string): boolean { + return ( + typeof error === "object" && error !== null && "name" in error && (error as Record<string, unknown>).name === name + ) + } + + static create<Name extends string, Data extends z.core.$ZodType>(name: Name, data: Data) { + const schema = z + .object({ + name: z.literal(name), + data, + }) + .meta({ + ref: name, + }) + const result = class extends NamedError { + public static readonly Schema = schema + + public override readonly name = name as Name + + constructor( + public readonly data: z.input<Data>, + options?: ErrorOptions, + ) { + super(name, options) + this.name = name + } + + static isInstance(input: any): input is InstanceType<typeof result> { + return typeof input === "object" && "name" in input && input.name === name + } + + schema() { + return schema + } + + toObject() { + return { + name: name, + data: this.data, + } + } + } + Object.defineProperty(result, "name", { value: name }) + return result + } + + public static readonly Unknown = NamedError.create( + "UnknownError", + z.object({ + message: z.string(), + }), + ) +} diff --git a/packages/core/src/util/flock.ts b/packages/core/src/util/flock.ts new file mode 100644 index 000000000..958bd9fd1 --- /dev/null +++ b/packages/core/src/util/flock.ts @@ -0,0 +1,358 @@ +import path from "path" +import os from "os" +import { randomBytes, randomUUID } from "crypto" +import { mkdir, readFile, rm, stat, utimes, writeFile } from "fs/promises" +import { Hash } from "./hash" +import { Effect } from "effect" + +export type FlockGlobal = { + state: string +} + +export namespace Flock { + let global: FlockGlobal | undefined + + export function setGlobal(g: FlockGlobal) { + global = g + } + + const root = () => { + if (!global) throw new Error("Flock global not set") + return path.join(global.state, "locks") + } + + // Defaults for callers that do not provide timing options. + const defaultOpts = { + staleMs: 60_000, + timeoutMs: 5 * 60_000, + baseDelayMs: 100, + maxDelayMs: 2_000, + } + + export interface WaitEvent { + key: string + attempt: number + delay: number + waited: number + } + + export type Wait = (input: WaitEvent) => void | Promise<void> + + export interface Options { + dir?: string + signal?: AbortSignal + staleMs?: number + timeoutMs?: number + baseDelayMs?: number + maxDelayMs?: number + onWait?: Wait + } + + type Opts = { + staleMs: number + timeoutMs: number + baseDelayMs: number + maxDelayMs: number + } + + type Owned = { + acquired: true + startHeartbeat: (intervalMs?: number) => void + release: () => Promise<void> + } + + export interface Lease { + release: () => Promise<void> + [Symbol.asyncDispose]: () => Promise<void> + } + + function code(err: unknown) { + if (typeof err !== "object" || err === null || !("code" in err)) return + const value = err.code + if (typeof value !== "string") return + return value + } + + function sleep(ms: number, signal?: AbortSignal) { + return new Promise<void>((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason ?? new Error("Aborted")) + return + } + + let timer: NodeJS.Timeout | undefined + + const done = () => { + signal?.removeEventListener("abort", abort) + resolve() + } + + const abort = () => { + if (timer) { + clearTimeout(timer) + } + signal?.removeEventListener("abort", abort) + reject(signal?.reason ?? new Error("Aborted")) + } + + signal?.addEventListener("abort", abort, { once: true }) + timer = setTimeout(done, ms) + }) + } + + function jitter(ms: number) { + const j = Math.floor(ms * 0.3) + const d = Math.floor(Math.random() * (2 * j + 1)) - j + return Math.max(0, ms + d) + } + + function mono() { + return performance.now() + } + + function wall() { + return performance.timeOrigin + mono() + } + + async function stats(file: string) { + try { + return await stat(file) + } catch (err) { + const errCode = code(err) + if (errCode === "ENOENT" || errCode === "ENOTDIR") return + throw err + } + } + + async function stale(lockDir: string, heartbeatPath: string, metaPath: string, staleMs: number) { + // Stale detection allows automatic recovery after crashed owners. + const now = wall() + const heartbeat = await stats(heartbeatPath) + if (heartbeat) { + return now - heartbeat.mtimeMs > staleMs + } + + const meta = await stats(metaPath) + if (meta) { + return now - meta.mtimeMs > staleMs + } + + const dir = await stats(lockDir) + if (!dir) { + return false + } + + return now - dir.mtimeMs > staleMs + } + + async function tryAcquireLockDir(lockDir: string, opts: Opts): Promise<Owned | { acquired: false }> { + const token = randomUUID?.() ?? randomBytes(16).toString("hex") + const metaPath = path.join(lockDir, "meta.json") + const heartbeatPath = path.join(lockDir, "heartbeat") + + try { + await mkdir(lockDir, { mode: 0o700 }) + } catch (err) { + if (code(err) !== "EEXIST") { + throw err + } + + if (!(await stale(lockDir, heartbeatPath, metaPath, opts.staleMs))) { + return { acquired: false } + } + + const breakerPath = lockDir + ".breaker" + try { + await mkdir(breakerPath, { mode: 0o700 }) + } catch (claimErr) { + const errCode = code(claimErr) + if (errCode === "EEXIST") { + const breaker = await stats(breakerPath) + if (breaker && wall() - breaker.mtimeMs > opts.staleMs) { + await rm(breakerPath, { recursive: true, force: true }).catch(() => undefined) + } + return { acquired: false } + } + + if (errCode === "ENOENT" || errCode === "ENOTDIR") { + return { acquired: false } + } + + throw claimErr + } + + try { + // Breaker ownership ensures only one contender performs stale cleanup. + if (!(await stale(lockDir, heartbeatPath, metaPath, opts.staleMs))) { + return { acquired: false } + } + + await rm(lockDir, { recursive: true, force: true }) + + try { + await mkdir(lockDir, { mode: 0o700 }) + } catch (retryErr) { + const errCode = code(retryErr) + if (errCode === "EEXIST" || errCode === "ENOTEMPTY") { + return { acquired: false } + } + throw retryErr + } + } finally { + await rm(breakerPath, { recursive: true, force: true }).catch(() => undefined) + } + } + + const meta = { + token, + pid: process.pid, + hostname: os.hostname(), + createdAt: new Date().toISOString(), + } + + await writeFile(heartbeatPath, "", { flag: "wx" }).catch(async () => { + await rm(lockDir, { recursive: true, force: true }) + throw new Error("Lock acquired but heartbeat already existed (possible compromise).") + }) + + await writeFile(metaPath, JSON.stringify(meta, null, 2), { flag: "wx" }).catch(async () => { + await rm(lockDir, { recursive: true, force: true }) + throw new Error("Lock acquired but meta.json already existed (possible compromise).") + }) + + let timer: NodeJS.Timeout | undefined + + const startHeartbeat = (intervalMs = Math.max(100, Math.floor(opts.staleMs / 3))) => { + if (timer) return + // Heartbeat prevents long critical sections from being evicted as stale. + timer = setInterval(() => { + const t = new Date() + void utimes(heartbeatPath, t, t).catch(() => undefined) + }, intervalMs) + timer.unref?.() + } + + const release = async () => { + if (timer) { + clearInterval(timer) + timer = undefined + } + + const current = await readFile(metaPath, "utf8") + .then((raw) => { + const parsed = JSON.parse(raw) + if (!parsed || typeof parsed !== "object") return {} + return { + token: "token" in parsed && typeof parsed.token === "string" ? parsed.token : undefined, + } + }) + .catch((err) => { + const errCode = code(err) + if (errCode === "ENOENT" || errCode === "ENOTDIR") { + throw new Error("Refusing to release: lock is compromised (metadata missing).") + } + if (err instanceof SyntaxError) { + throw new Error("Refusing to release: lock is compromised (metadata invalid).") + } + throw err + }) + // Token check prevents deleting a lock that was re-acquired by another process. + if (current.token !== token) { + throw new Error("Refusing to release: lock token mismatch (not the owner).") + } + + await rm(lockDir, { recursive: true, force: true }) + } + + return { + acquired: true, + startHeartbeat, + release, + } + } + + async function acquireLockDir( + lockDir: string, + input: { key: string; onWait?: Wait; signal?: AbortSignal }, + opts: Opts, + ) { + const stop = mono() + opts.timeoutMs + let attempt = 0 + let waited = 0 + let delay = opts.baseDelayMs + + while (true) { + input.signal?.throwIfAborted() + + const res = await tryAcquireLockDir(lockDir, opts) + if (res.acquired) { + return res + } + + if (mono() > stop) { + throw new Error(`Timed out waiting for lock: ${input.key}`) + } + + attempt += 1 + const ms = jitter(delay) + await input.onWait?.({ + key: input.key, + attempt, + delay: ms, + waited, + }) + await sleep(ms, input.signal) + waited += ms + delay = Math.min(opts.maxDelayMs, Math.floor(delay * 1.7)) + } + } + + export async function acquire(key: string, input: Options = {}): Promise<Lease> { + input.signal?.throwIfAborted() + const cfg: Opts = { + staleMs: input.staleMs ?? defaultOpts.staleMs, + timeoutMs: input.timeoutMs ?? defaultOpts.timeoutMs, + baseDelayMs: input.baseDelayMs ?? defaultOpts.baseDelayMs, + maxDelayMs: input.maxDelayMs ?? defaultOpts.maxDelayMs, + } + const dir = input.dir ?? root() + + await mkdir(dir, { recursive: true }) + const lockfile = path.join(dir, Hash.fast(key) + ".lock") + const lock = await acquireLockDir( + lockfile, + { + key, + onWait: input.onWait, + signal: input.signal, + }, + cfg, + ) + lock.startHeartbeat() + + const release = () => lock.release() + return { + release, + [Symbol.asyncDispose]() { + return release() + }, + } + } + + export async function withLock<T>(key: string, fn: () => Promise<T>, input: Options = {}) { + await using _ = await acquire(key, input) + input.signal?.throwIfAborted() + return await fn() + } + + export const effect = Effect.fn("Flock.effect")(function* (key: string, input: Options = {}) { + return yield* Effect.acquireRelease( + Effect.promise((signal) => Flock.acquire(key, { ...input, signal })).pipe( + Effect.withSpan("Flock.acquire", { + attributes: { key }, + }), + ), + (lock) => Effect.promise(() => lock.release()).pipe(Effect.withSpan("Flock.release")), + ).pipe(Effect.asVoid) + }) +} diff --git a/packages/core/src/util/fn.ts b/packages/core/src/util/fn.ts new file mode 100644 index 000000000..9efe4622f --- /dev/null +++ b/packages/core/src/util/fn.ts @@ -0,0 +1,11 @@ +import { z } from "zod" + +export function fn<T extends z.ZodType, Result>(schema: T, cb: (input: z.infer<T>) => Result) { + const result = (input: z.infer<T>) => { + const parsed = schema.parse(input) + return cb(parsed) + } + result.force = (input: z.infer<T>) => cb(input) + result.schema = schema + return result +} diff --git a/packages/core/src/util/glob.ts b/packages/core/src/util/glob.ts new file mode 100644 index 000000000..febf062da --- /dev/null +++ b/packages/core/src/util/glob.ts @@ -0,0 +1,34 @@ +import { glob, globSync, type GlobOptions } from "glob" +import { minimatch } from "minimatch" + +export namespace Glob { + export interface Options { + cwd?: string + absolute?: boolean + include?: "file" | "all" + dot?: boolean + symlink?: boolean + } + + function toGlobOptions(options: Options): GlobOptions { + return { + cwd: options.cwd, + absolute: options.absolute, + dot: options.dot, + follow: options.symlink ?? false, + nodir: options.include !== "all", + } + } + + export async function scan(pattern: string, options: Options = {}): Promise<string[]> { + return glob(pattern, toGlobOptions(options)) as Promise<string[]> + } + + export function scanSync(pattern: string, options: Options = {}): string[] { + return globSync(pattern, toGlobOptions(options)) as string[] + } + + export function match(pattern: string, filepath: string): boolean { + return minimatch(filepath, pattern, { dot: true }) + } +} diff --git a/packages/core/src/util/hash.ts b/packages/core/src/util/hash.ts new file mode 100644 index 000000000..680e0f40b --- /dev/null +++ b/packages/core/src/util/hash.ts @@ -0,0 +1,7 @@ +import { createHash } from "crypto" + +export namespace Hash { + export function fast(input: string | Buffer): string { + return createHash("sha1").update(input).digest("hex") + } +} diff --git a/packages/core/src/util/identifier.ts b/packages/core/src/util/identifier.ts new file mode 100644 index 000000000..ba28a351b --- /dev/null +++ b/packages/core/src/util/identifier.ts @@ -0,0 +1,48 @@ +import { randomBytes } from "crypto" + +export namespace Identifier { + const LENGTH = 26 + + // State for monotonic ID generation + let lastTimestamp = 0 + let counter = 0 + + export function ascending() { + return create(false) + } + + export function descending() { + return create(true) + } + + function randomBase62(length: number): string { + const chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + let result = "" + const bytes = randomBytes(length) + for (let i = 0; i < length; i++) { + result += chars[bytes[i] % 62] + } + return result + } + + export function create(descending: boolean, timestamp?: number): string { + const currentTimestamp = timestamp ?? Date.now() + + if (currentTimestamp !== lastTimestamp) { + lastTimestamp = currentTimestamp + counter = 0 + } + counter++ + + let now = BigInt(currentTimestamp) * BigInt(0x1000) + BigInt(counter) + + now = descending ? ~now : now + + const timeBytes = Buffer.alloc(6) + for (let i = 0; i < 6; i++) { + timeBytes[i] = Number((now >> BigInt(40 - 8 * i)) & BigInt(0xff)) + } + + return timeBytes.toString("hex") + randomBase62(LENGTH - 12) + } +} diff --git a/packages/core/src/util/iife.ts b/packages/core/src/util/iife.ts new file mode 100644 index 000000000..ca9ae6c10 --- /dev/null +++ b/packages/core/src/util/iife.ts @@ -0,0 +1,3 @@ +export function iife<T>(fn: () => T) { + return fn() +} diff --git a/packages/core/src/util/lazy.ts b/packages/core/src/util/lazy.ts new file mode 100644 index 000000000..935ebe0f9 --- /dev/null +++ b/packages/core/src/util/lazy.ts @@ -0,0 +1,11 @@ +export function lazy<T>(fn: () => T) { + let value: T | undefined + let loaded = false + + return (): T => { + if (loaded) return value as T + loaded = true + value = fn() + return value as T + } +} diff --git a/packages/core/src/util/module.ts b/packages/core/src/util/module.ts new file mode 100644 index 000000000..6ed3b23d7 --- /dev/null +++ b/packages/core/src/util/module.ts @@ -0,0 +1,10 @@ +import { createRequire } from "node:module" +import path from "node:path" + +export namespace Module { + export function resolve(id: string, dir: string) { + try { + return createRequire(path.join(dir, "package.json")).resolve(id) + } catch {} + } +} diff --git a/packages/core/src/util/path.ts b/packages/core/src/util/path.ts new file mode 100644 index 000000000..b87316358 --- /dev/null +++ b/packages/core/src/util/path.ts @@ -0,0 +1,37 @@ +export function getFilename(path: string | undefined) { + if (!path) return "" + const trimmed = path.replace(/[/\\]+$/, "") + const parts = trimmed.split(/[/\\]/) + return parts[parts.length - 1] ?? "" +} + +export function getDirectory(path: string | undefined) { + if (!path) return "" + const trimmed = path.replace(/[/\\]+$/, "") + const parts = trimmed.split(/[/\\]/) + return parts.slice(0, parts.length - 1).join("/") + "/" +} + +export function getFileExtension(path: string | undefined) { + if (!path) return "" + const parts = path.split(".") + return parts[parts.length - 1] +} + +export function getFilenameTruncated(path: string | undefined, maxLength: number = 20) { + const filename = getFilename(path) + if (filename.length <= maxLength) return filename + const lastDot = filename.lastIndexOf(".") + const ext = lastDot <= 0 ? "" : filename.slice(lastDot) + const available = maxLength - ext.length - 1 // -1 for ellipsis + if (available <= 0) return filename.slice(0, maxLength - 1) + "…" + return filename.slice(0, available) + "…" + ext +} + +export function truncateMiddle(text: string, maxLength: number = 20) { + if (text.length <= maxLength) return text + const available = maxLength - 1 // -1 for ellipsis + const start = Math.ceil(available / 2) + const end = Math.floor(available / 2) + return text.slice(0, start) + "…" + text.slice(-end) +} diff --git a/packages/core/src/util/retry.ts b/packages/core/src/util/retry.ts new file mode 100644 index 000000000..831d23800 --- /dev/null +++ b/packages/core/src/util/retry.ts @@ -0,0 +1,42 @@ +export interface RetryOptions { + attempts?: number + delay?: number + factor?: number + maxDelay?: number + retryIf?: (error: unknown) => boolean +} + +const TRANSIENT_MESSAGES = [ + "load failed", + "network connection was lost", + "network request failed", + "failed to fetch", + "econnreset", + "econnrefused", + "etimedout", + "socket hang up", +] + +function isTransientError(error: unknown): boolean { + if (!error) return false + // oxlint-disable-next-line no-base-to-string -- error is unknown, intentional coercion for message matching + const message = String(error instanceof Error ? error.message : error).toLowerCase() + return TRANSIENT_MESSAGES.some((m) => message.includes(m)) +} + +export async function retry<T>(fn: () => Promise<T>, options: RetryOptions = {}): Promise<T> { + const { attempts = 3, delay = 500, factor = 2, maxDelay = 10000, retryIf = isTransientError } = options + + let lastError: unknown + for (let attempt = 0; attempt < attempts; attempt++) { + try { + return await fn() + } catch (error) { + lastError = error + if (attempt === attempts - 1 || !retryIf(error)) throw error + const wait = Math.min(delay * Math.pow(factor, attempt), maxDelay) + await new Promise((resolve) => setTimeout(resolve, wait)) + } + } + throw lastError +} diff --git a/packages/core/src/util/slug.ts b/packages/core/src/util/slug.ts new file mode 100644 index 000000000..62cf0e57b --- /dev/null +++ b/packages/core/src/util/slug.ts @@ -0,0 +1,74 @@ +export namespace Slug { + const ADJECTIVES = [ + "brave", + "calm", + "clever", + "cosmic", + "crisp", + "curious", + "eager", + "gentle", + "glowing", + "happy", + "hidden", + "jolly", + "kind", + "lucky", + "mighty", + "misty", + "neon", + "nimble", + "playful", + "proud", + "quick", + "quiet", + "shiny", + "silent", + "stellar", + "sunny", + "swift", + "tidy", + "witty", + ] as const + + const NOUNS = [ + "cabin", + "cactus", + "canyon", + "circuit", + "comet", + "eagle", + "engine", + "falcon", + "forest", + "garden", + "harbor", + "island", + "knight", + "lagoon", + "meadow", + "moon", + "mountain", + "nebula", + "orchid", + "otter", + "panda", + "pixel", + "planet", + "river", + "rocket", + "sailor", + "squid", + "star", + "tiger", + "wizard", + "wolf", + ] as const + + export function create() { + return [ + ADJECTIVES[Math.floor(Math.random() * ADJECTIVES.length)], + NOUNS[Math.floor(Math.random() * NOUNS.length)], + ].join("-") + } +} |
