From aa63d6b9f4cb14535fd0191ab358c27f0ee62ce8 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Mon, 19 Oct 2026 23:48:50 +0900 Subject: feat(markdown): render local-path images inline via backend image route --- PLAN.md | 85 +++++++++++++++++ backend-handoff.md | 24 +++++ src/app/views/ChatPane.svelte | 1 + src/features/chat/ui.test.ts | 19 ++++ src/features/chat/ui/ChatView.svelte | 13 ++- src/features/markdown/logic/markdown.test.ts | 93 +++++++++++++++++- src/features/markdown/logic/markdown.ts | 110 ++++++++++++++++++++-- src/features/markdown/logic/streaming-markdown.ts | 61 ++++++++++-- src/features/markdown/ui/Markdown.svelte | 19 +++- 9 files changed, 404 insertions(+), 21 deletions(-) create mode 100644 PLAN.md diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..c45994d --- /dev/null +++ b/PLAN.md @@ -0,0 +1,85 @@ +# Feature: local-image-markdown + +**Goal:** Agents can show the user images in chat purely via Markdown — `![alt](/absolute/path/to/image.png)` or `file:///…` — with no tool call. The chat UI renders them inline. + +**Repos:** backend AND frontend (separate worktrees of the same feature branch `feature/local-image-markdown`; the API contract below is FIXED — do not change it unilaterally). + +**Out of scope / decisions already made:** +- **Raster images only** — png, jpg, jpeg, webp, gif, bmp. **SVG is intentionally excluded** (XSS vector via embedded scripts). +- No auth changes on the route (the API is unauthenticated today; the extension whitelist + magic-byte sniff bounds the exposure to actual image files). +- Stored custom system-prompt templates are NOT migrated — only `DEFAULT_TEMPLATE` changes. If a stored template exists it overrides the default; just note this in the summary, do not attempt migration. + +--- + +## Backend (packages: `transport-http`, `system-prompt`) + +### 1. New route `GET /local-image` in `packages/transport-http/src/app.ts` + +Place it next to the existing tmp-image route (`app.get("/images/:conversationId/:imageId", …)`), mirroring its style (dynamic `node:path` / `node:fs/promises` imports, same response shape). + +**Contract (pinned):** +- Query param: `path` — a URL-encoded absolute filesystem path to an image. + - Expand a leading `~` to the OS home dir (agents often write `~/…`). + - Reject with `400 { error: … }` if the decoded path is not absolute (after `~` expansion) or is empty. +- Extension whitelist (lowercase, after decode): `.png .jpg .jpeg .webp .gif .bmp` → MIME map (`image/png`, `image/jpeg`, `image/webp`, `image/gif`, `image/bmp`). Any other extension → `400`. +- **Magic-byte sniff** before serving (cheap header check per format, e.g. PNG `89 50 4E 47`, JPEG `FF D8 FF`, GIF `GIF8`, WEBP `RIFF…WEBP`, BMP `BM`). Mismatch → `400`. This guarantees the endpoint can never serve a non-image file regardless of extension. +- Read failure (missing/unreadable) → `404 { error: "Image not found" }` (match the existing route's shape). +- Success: `Response(buf, { headers: { "Content-Type": , "Cache-Control": "no-cache", "X-Content-Type-Options": "nosniff" } })`. +- The endpoint intentionally serves ANY absolute image path on the machine — that is the feature (the agent already has shell/read access to the same machine). Do not add a root allowlist. + +**Tests:** follow the existing transport-http app test conventions — 200 + correct MIME for a real temp image file per format, 400 for non-image extension, 400 for magic-byte mismatch (e.g. PNG bytes named `.png` vs text bytes named `.png`), 400 for relative path, 404 for missing file, `~` expansion. + +### 2. System-prompt default template — `packages/system-prompt/src/service.ts` + +Add to `DEFAULT_TEMPLATE` (after the cwd line) a paragraph telling agents the capability, e.g.: + +``` +You can show the user images directly in chat by embedding them in your Markdown +replies, e.g. ![description](/absolute/path/to/image.png). Local absolute paths +(and file:// URLs) are rendered inline by the chat UI; relative paths resolve +against the current working directory. Supported formats: png, jpg, jpeg, webp, +gif, bmp. No tool call is needed — just the Markdown image syntax. +``` + +(Exact wording may be polished; the capability, the syntax, and the format list must be present.) Update `service.test.ts` structure assertions if needed. + +--- + +## Frontend (feature: `src/features/markdown`, wiring in `src/features/chat`) + +### 3. Local-path image rewriting in `src/features/markdown/logic/markdown.ts` + +Today `![x](file:///…)` and `![x](/home/…)` render broken: DOMPurify strips `file:` srcs, and a root-relative `/home/…` src 404s against the frontend origin. Remote `http(s)` images already work and must keep working. + +**Required behavior — `renderMarkdown`, `renderMarkdownTail`, and `renderOpenFence` must all rewrite image srcs consistently (all three funnel through `sanitize()`):** + +For each `` produced by Markdown image syntax `![alt](src)`: +- `file:///path` → strip the scheme → local absolute path. +- `/absolute/path` → local absolute path. +- Relative path (no scheme, no leading `/`, not `data:`/`http(s)`) → resolve against the **conversation cwd** (threaded in as a render option; empty/absent cwd → leave as-is / no rewrite). +- `http(s)://…`, `data:` → leave untouched. + +Rewritten src = `` `${apiBase}/local-image?path=${encodeURIComponent(absPath)}` `` — reuse the join-without-double-slash semantics of `src/core/chunks/image-url.ts` (`apiBase` may be empty → root-relative `/local-image?…`). `apiBase` and `cwd` are per-conversation render options, not module constants — thread them through as an opts parameter (e.g. `{ image?: { apiBase: string; cwd?: string | null } }`) from the render entry points. Whether you implement this via a DOMPurify `afterSanitizeAttributes` IMG hook (the module already uses this pattern for link `target=_blank`) or a pre-parse rewrite of the Markdown source is your call — cover Markdown image syntax at minimum; raw-HTML `` tags in Markdown are best-effort. + +### 4. Component threading — `Markdown.svelte` + `ChatView.svelte` + +- `Markdown.svelte` (`src/features/markdown/ui/Markdown.svelte`): new optional prop carrying `{ apiBase: string; cwd?: string | null }`, forwarded into all render calls (including the streaming path via `advance`/`createStreamingMarkdown` — check `logic/streaming-markdown.ts` for how render opts flow). +- `ChatView.svelte` (renders `` around line 190): pass `apiBase` (the store already exposes the HTTP base, see how `resolveImageUrl` callers get it) and the conversation `cwd` (already reactive state in the app store — `$state cwd`, fetched via `GET /conversations/:id/cwd`). + +### 5. Tests + +Extend `src/features/markdown/logic/markdown.test.ts` (and streaming tests if the tail path is affected): +- `file:///home/u/pic.png` + apiBase `http://localhost:24203` → src `http://localhost:24203/local-image?path=%2Fhome%2Fu%2Fpic.png`. +- `/home/u/pic.png` with empty apiBase → root-relative `/local-image?path=…`. +- Relative `shot.png` with cwd `/w/foo` → `…/local-image?path=%2Fw%2Ffoo%2Fshot.png`; without cwd → untouched. +- `https://example.com/a.png` and `data:` URLs → untouched. +- Streaming tail render rewrites too. + +--- + +## Definition of done (both repos) + +- Backend: `bun install` then the repo's verification commands from AGENTS.md (typecheck / test / check / build) all green; route + template changes committed to `feature/local-image-markdown`. +- Frontend: same — all green, committed to `feature/local-image-markdown`. +- Manual smoke (report only, don't block): with the stack running, an agent message containing `![x](/abs/path.png)` shows the image inline. +- Do NOT merge or push. Commit to the feature branch only. diff --git a/backend-handoff.md b/backend-handoff.md index ab79bf2..1986e94 100644 --- a/backend-handoff.md +++ b/backend-handoff.md @@ -18,6 +18,30 @@ > ?archived=`) was made in the same branch and the FE consumes it via the `file:` deps (dists > rebuilt). Recorded below. +## Milestone: Local-path images in Markdown (`feature/local-image-markdown`, 2026-10-19) + +**Slice status:** COMPLETE (frontend half; the backend route is implemented in parallel in +`../backend` from the same pinned contract). + +- `features/markdown/logic/markdown.ts`: `renderMarkdown` / `renderMarkdownTail` accept per-conversation + `image: { apiBase, cwd? }`. A custom marked renderer rewrites Markdown image token hrefs: + `file://…` → decoded absolute path, `/abs/path` → itself, relative → joined to `cwd`, `~/…` passed + through for backend expansion; `http(s)://`, `data:`, and other schemes are untouched. Local paths + become `${apiBase}/local-image?path=` via `resolveImageUrl` (empty `apiBase` → + root-relative). Rewriting happens on the token stream, so fenced/inline code is never touched. +- `logic/streaming-markdown.ts`: `advance(state, text, opts)` threads the same options through the + settled, tail, and open-fence pipelines. The cache is keyed on `apiBase`/`cwd` so an async cwd load + invalidates and re-renders settled blocks; string identity is preserved while the options are stable. +- `ui/Markdown.svelte`: new optional `image` prop. `ChatView` gains `cwd` and passes + `image={{ apiBase: apiBaseUrl, cwd }}`; `ChatPane` supplies `store.httpBase` + `store.cwd`. +- Tests: `file://`, absolute, relative-with/without-cwd, empty-`apiBase`, http(s), `data:`, fenced-code + exemption, streaming tail + settled prefix + options-change invalidation, and ChatView threading. +- Backend contract: expects `GET /local-image?path=` exactly as pinned in `PLAN.md` (absolute path, + `~` expansion, raster whitelist, magic-byte sniff). No FE contract/type changes; **no backend asks**. +- Verification: typecheck **0 errors / 0 warnings**; tests **2,070 passed** (114 files); + `bun run check` clean; build succeeds (existing bundle-size warning only). No wire/browser-effect + adapter changes → no live probe required for this slice; manual inline-image smoke remains user-run. + ## Milestone: Agent tab grid + subagent groups (`feature/agent-tab-grid`, 2026-10-15) - FE: the tabs view is now a 2-column grid of identical fixed-height cards (`TabCard`); a tab with visible subagents is double-width and wrapped with them in a darker, nestable box diff --git a/src/app/views/ChatPane.svelte b/src/app/views/ChatPane.svelte index 31e719e..d1630af 100644 --- a/src/app/views/ChatPane.svelte +++ b/src/app/views/ChatPane.svelte @@ -86,6 +86,7 @@ thinkingKeyBase={store.activeChat.thinkingKeyBase} providerRetry={store.activeChat.providerRetry} apiBaseUrl={store.httpBase} + cwd={store.cwd} {cacheTimer} /> {/key} diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index 10af76c..c557e4f 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -56,6 +56,25 @@ describe("ChatView", () => { expect(screen.getByText("Hello!")).toBeInTheDocument(); }); + it("renders local-path Markdown images through the backend image route", () => { + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "assistant", + chunk: { type: "text", text: "![pic](/home/u/pic.png)" }, + provisional: false, + }, + ]; + + const { container } = render(ChatView, { + props: { chunks, apiBaseUrl: "http://localhost:24203", cwd: "/w/foo" }, + }); + + expect(container.querySelector("img")?.getAttribute("src")).toBe( + "http://localhost:24203/local-image?path=%2Fhome%2Fu%2Fpic.png", + ); + }); + it("shows the show-earlier button only when earlier history is unloaded, and pages it in", async () => { const chunks: RenderedChunk[] = [ { seq: 26, role: "user", chunk: { type: "text", text: "later" }, provisional: false }, diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte index 4fd0911..f34de9a 100644 --- a/src/features/chat/ui/ChatView.svelte +++ b/src/features/chat/ui/ChatView.svelte @@ -30,6 +30,7 @@ thinkingKeyBase = 0, providerRetry = null, apiBaseUrl = "", + cwd = null, cacheTimer = null, }: { chunks: readonly RenderedChunk[]; @@ -59,6 +60,12 @@ * "" (root-relative — a browser resolves `/images/…` against its origin). */ apiBaseUrl?: string; + /** + * The active conversation's working directory (null when unset). Relative + * Markdown image srcs (`![x](shot.png)`) resolve against it; local absolute + * and `file://` paths do not need it. Passed to `Markdown` with `apiBaseUrl`. + */ + cwd?: string | null; /** * The cache timer (gaps between model requests): a running step gap shows * live at the bottom (under the executing tools), a running turn gap live @@ -187,7 +194,11 @@
{#if rendered.chunk.type === "text"} - + {:else if rendered.chunk.type === "error"}