From 72663945ef0a0053fbfd83b168e90f12169a5c3a Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Fri, 18 Sep 2026 18:39:00 +0900 Subject: feat(chat): compress images at attach time (JPEG re-encode + rescale) with a per-image toggle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Staged images now default to a compressed variant: LOSSLESS sources (PNG/BMP) are re-encoded to JPEG at q0.85, rescaled to cap the longest edge at 2048px (never upscaled, high-quality smoothing, alpha flattened onto white), and used only under a smaller-wins rule. JPEG/WebP/GIF are sent verbatim. Each staged thumbnail gets a Compress checkmark (default ON, reversible until send, cached so flipping never re-encodes); OFF sends the original untouched bytes. Conversion runs async at staging and any failure silently falls back to the original — sending is never blocked or broken. Pure decision logic lives in features/chat/logic/image-compress.ts (format selection, dimension math, smaller-wins); the canvas encode is a thin injected edge (features/chat/adapter/image-encode.ts, Composer prop encodeJpeg). No wire-contract, send-path, or backend changes; complements the backend's body-budget eviction safety net by shrinking payloads at the source. Verification: typecheck 0 errors, vitest 1720/1720 (twice), biome clean, build succeeds. New tests: 12 pure-logic, 2 encoder-edge, 6 component. --- backend-handoff.md | 23 +++++ src/features/chat/adapter/image-encode.test.ts | 20 ++++ src/features/chat/adapter/image-encode.ts | 67 ++++++++++++++ src/features/chat/logic/image-compress.test.ts | 76 +++++++++++++++ src/features/chat/logic/image-compress.ts | 77 ++++++++++++++++ src/features/chat/ui.test.ts | 122 +++++++++++++++++++++++++ src/features/chat/ui/Composer.svelte | 116 +++++++++++++++++++++-- 7 files changed, 493 insertions(+), 8 deletions(-) create mode 100644 src/features/chat/adapter/image-encode.test.ts create mode 100644 src/features/chat/adapter/image-encode.ts create mode 100644 src/features/chat/logic/image-compress.test.ts create mode 100644 src/features/chat/logic/image-compress.ts diff --git a/backend-handoff.md b/backend-handoff.md index c15db17..73aa011 100644 --- a/backend-handoff.md +++ b/backend-handoff.md @@ -305,3 +305,26 @@ changes, NO contract changes — the wire types and every endpoint consumed are `tile-layout.test.ts` order assertions — coordinates unchanged), biome clean on touched files, `build` succeeds. Browser-only (pointer capture) — needs a human at the page to confirm the three fixes live (see commit message checklist). + +## 2026-09-18 — feature/image-attach-compression (FE-only) + +- **Attach-time image compression** in the chat composer: every staged image now carries a + per-image `Compress` toggle (default ON, next to the remove button on the thumbnail, + tooltip "PNG → JPEG + resize to 2048px"). ON re-encodes LOSSLESS sources (PNG/BMP) to + JPEG q0.85, rescales to cap the longest edge at 2048px (never upscales, high-quality + smoothing, alpha flattened onto white), and uses the result only under a smaller-wins + rule. JPEG/WebP/GIF are sent verbatim (toggle hidden — recompressing lossy/animated + images degrades them and saves nothing). OFF sends the original untouched bytes; the + conversion is cached so flipping the toggle never re-encodes. Conversion is async at + staging (the original stages immediately) and ANY failure silently falls back to the + original — sending can never be blocked or broken by compression. +- **NO wire-contract changes.** `ImageInput`/`ImageChunk` are untouched; only the bytes a + client chooses to put in `ChatRequest.images` change (still base64 data URLs, now + sometimes `data:image/jpeg;base64,…` for a source that was `image/png`). No backend ask. + This complements the backend's body-budget eviction safety net by shrinking images at + the source so that path rarely triggers. +- Verification: `typecheck` 0 errors/0 warnings; vitest 1720/1720 (twice — incl. new pure + logic tests for format selection/dimension math/smaller-wins + component tests for the + toggle default/reversibility/caching/fallback/mimeType); biome clean; `build` succeeds. + Manual browser sanity (paste a ~3MB PNG → staged size drops; uncheck → original; + re-check → no visible re-encode; send works) still needs a human at the page. diff --git a/src/features/chat/adapter/image-encode.test.ts b/src/features/chat/adapter/image-encode.test.ts new file mode 100644 index 0000000..dbaa8ae --- /dev/null +++ b/src/features/chat/adapter/image-encode.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { convertToJpeg } from "./image-encode"; + +describe("convertToJpeg (canvas edge — real environment, no mocks)", () => { + it("skips non-convertible formats without touching the canvas", async () => { + const jpeg = new File([new Uint8Array([0xff, 0xd8])], "photo.jpg", { type: "image/jpeg" }); + await expect(convertToJpeg(jpeg)).resolves.toBe(null); + const gif = new File([new Uint8Array([0x47])], "anim.gif", { type: "image/gif" }); + await expect(convertToJpeg(gif)).resolves.toBe(null); + const unnamed = new File([new Uint8Array([0x00])], "blob", { type: "" }); + await expect(convertToJpeg(unnamed)).resolves.toBe(null); + }); + + it("falls back to null on decode failure (no real canvas in this environment)", async () => { + const png = new File([new Uint8Array([0x89, 0x50])], "shot.png", { type: "image/png" }); + // jsdom has no `createImageBitmap` → the decode throws → the failure path + // must resolve to `null` (never reject), so sending is never blocked. + await expect(convertToJpeg(png)).resolves.toBe(null); + }); +}); diff --git a/src/features/chat/adapter/image-encode.ts b/src/features/chat/adapter/image-encode.ts new file mode 100644 index 0000000..4570a5a --- /dev/null +++ b/src/features/chat/adapter/image-encode.ts @@ -0,0 +1,67 @@ +import { + computeResizedDimensions, + JPEG_QUALITY, + shouldConvertImage, + useConvertedImage, +} from "../logic/image-compress"; + +/** A successfully converted image: a JPEG blob at `JPEG_QUALITY`. */ +export interface ConvertedJpeg { + readonly blob: Blob; + readonly mimeType: "image/jpeg"; +} + +/** + * The injected browser edge for attach-time compression: a `File` in, a + * (maybe) smaller JPEG out. Returns `null` when the source is not converted + * (non-convertible format) or the conversion is skipped/failed (decode or + * encode error, smaller-wins rule) — the caller then sends the original + * bytes. Compression must never block or break sending, so every failure is + * swallowed into `null`. + */ +export type ImageJpegEncoder = (file: File) => Promise; + +/** + * Canvas-based JPEG re-encode: decode the source, rescale to cap the longest + * edge at `MAX_IMAGE_EDGE_PX` (never upscale, high-quality smoothing), flatten + * any transparency onto a white background (JPEG has no alpha), and encode at + * `JPEG_QUALITY`. Applied only to convertible (lossless) sources; the result + * is used only under the smaller-wins rule. + * + * Thin by design — every decision comes from `../logic/image-compress`. + */ +export const convertToJpeg: ImageJpegEncoder = async (file: File) => { + try { + if (!shouldConvertImage(file.type)) return null; + + const bitmap = await createImageBitmap(file); + const target = computeResizedDimensions(bitmap.width, bitmap.height); + + const canvas = document.createElement("canvas"); + canvas.width = target.width; + canvas.height = target.height; + const ctx = canvas.getContext("2d"); + if (ctx === null) { + bitmap.close(); + return null; + } + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = "high"; + // Flatten transparency: JPEG has no alpha channel. + ctx.fillStyle = "#ffffff"; + ctx.fillRect(0, 0, target.width, target.height); + ctx.drawImage(bitmap, 0, 0, target.width, target.height); + bitmap.close(); + + const blob = await new Promise((resolve) => { + canvas.toBlob(resolve, "image/jpeg", JPEG_QUALITY); + }); + if (blob === null) return null; + // Smaller-wins: keep the original when the re-encode didn't actually help. + if (!useConvertedImage(file.size, blob.size)) return null; + return { blob, mimeType: "image/jpeg" }; + } catch { + // Decode/encode failure → the original image is sent untouched. + return null; + } +}; diff --git a/src/features/chat/logic/image-compress.test.ts b/src/features/chat/logic/image-compress.test.ts new file mode 100644 index 0000000..88726bf --- /dev/null +++ b/src/features/chat/logic/image-compress.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { + computeResizedDimensions, + MAX_IMAGE_EDGE_PX, + shouldConvertImage, + useConvertedImage, +} from "./image-compress"; + +describe("shouldConvertImage", () => { + it("converts lossless sources (PNG, BMP)", () => { + expect(shouldConvertImage("image/png")).toBe(true); + expect(shouldConvertImage("image/bmp")).toBe(true); + }); + + it("skips already-lossy and animated formats", () => { + expect(shouldConvertImage("image/jpeg")).toBe(false); + expect(shouldConvertImage("image/webp")).toBe(false); + expect(shouldConvertImage("image/gif")).toBe(false); + }); + + it("leaves unknown or absent MIME types alone", () => { + expect(shouldConvertImage(undefined)).toBe(false); + expect(shouldConvertImage("")).toBe(false); + expect(shouldConvertImage("image/tiff")).toBe(false); + expect(shouldConvertImage("image/svg+xml")).toBe(false); + }); +}); + +describe("computeResizedDimensions", () => { + it("never upscales a source already within the cap", () => { + expect(computeResizedDimensions(100, 50)).toEqual({ width: 100, height: 50 }); + expect(computeResizedDimensions(2048, 1024)).toEqual({ width: 2048, height: 1024 }); + expect(computeResizedDimensions(640, 640)).toEqual({ width: 640, height: 640 }); + }); + + it("caps the longest edge at 2048", () => { + expect(computeResizedDimensions(4096, 2048)).toEqual({ width: 2048, height: 1024 }); + expect(computeResizedDimensions(8192, 4096)).toEqual({ width: 2048, height: 1024 }); + }); + + it("caps the longest edge regardless of orientation (portrait)", () => { + expect(computeResizedDimensions(2000, 3000)).toEqual({ width: 1365, height: 2048 }); + }); + + it("preserves aspect ratio on the rescale", () => { + const { width, height } = computeResizedDimensions(3000, 2000); + expect(width).toBe(2048); + // 2000 * (2048/3000) = 1365.33… → 1365 + expect(height).toBe(1365); + expect(width / height).toBeCloseTo(3000 / 2000, 2); + }); + + it("handles extreme aspect ratios without exceeding the cap", () => { + expect(computeResizedDimensions(5000, 10)).toEqual({ width: 2048, height: 4 }); + expect(computeResizedDimensions(10, 5000)).toEqual({ width: 4, height: 2048 }); + }); + + it("honors a custom cap", () => { + expect(computeResizedDimensions(1000, 500, 250)).toEqual({ width: 250, height: 125 }); + }); + + it("uses the default 2048px cap constant", () => { + expect(MAX_IMAGE_EDGE_PX).toBe(2048); + }); +}); + +describe("useConvertedImage", () => { + it("uses the conversion only when it is strictly fewer bytes", () => { + expect(useConvertedImage(3_000_000, 300_000)).toBe(true); + }); + + it("keeps the original when equal or larger", () => { + expect(useConvertedImage(300_000, 300_000)).toBe(false); + expect(useConvertedImage(100_000, 300_000)).toBe(false); + }); +}); diff --git a/src/features/chat/logic/image-compress.ts b/src/features/chat/logic/image-compress.ts new file mode 100644 index 0000000..a9037b8 --- /dev/null +++ b/src/features/chat/logic/image-compress.ts @@ -0,0 +1,77 @@ +/** + * Pure decision logic for attach-time image compression (the `Compress` + * toggle on the composer's staged-image thumbnails). + * + * The pipeline (re-encode to JPEG + rescale) runs at the browser edge + * (`../adapter/image-encode`); this module owns every DECISION so it stays + * pure `input → output` and unit-testable without a DOM: + * + * - `shouldConvertImage` — which source formats get converted. + * - `computeResizedDimensions` — the rescale math (cap the longest edge, + * never upscale, preserve aspect). + * - `useConvertedImage` — the smaller-wins rule. + */ + +/** JPEG encode quality for the canvas re-encode. */ +export const JPEG_QUALITY = 0.85; + +/** Longest-edge cap (px) for the rescale. Never upscale past the source. */ +export const MAX_IMAGE_EDGE_PX = 2048; + +/** + * Source formats that get converted. Only LOSSLESS sources are re-encoded: + * PNG (the screenshot case — multi-MB PNGs are the reason this exists) plus + * BMP (trivially canvas-decodable in every browser). Already-lossy formats + * (JPEG/WebP) and animated ones (GIF) are sent verbatim — recompressing them + * degrades quality and saves nothing, and re-encoding a GIF destroys its + * animation. TIFF is NOT listed: browsers can't natively decode it, so the + * canvas path would always fall back to the original anyway. An oversized + * JPEG stays untouched (the backend's body-budget safety net handles the + * extreme case). + */ +const CONVERTIBLE_MIME_TYPES = new Set(["image/png", "image/bmp"]); + +/** + * Whether a source MIME type is converted to JPEG at staging time. + * Unknown/absent MIME types are left alone (conservative — never gamble on + * bytes we can't classify). + */ +export function shouldConvertImage(mimeType: string | undefined): boolean { + return mimeType !== undefined && CONVERTIBLE_MIME_TYPES.has(mimeType); +} + +/** Pixel dimensions of a raster image. */ +export interface Dimensions { + readonly width: number; + readonly height: number; +} + +/** + * Target dimensions after the rescale: cap the LONGEST edge at `maxEdge` + * (default `MAX_IMAGE_EDGE_PX`), never upscale (a smaller source passes + * through unchanged), and preserve aspect ratio. + * + * Pure math — the canvas draw itself lives at the browser edge. + */ +export function computeResizedDimensions( + width: number, + height: number, + maxEdge: number = MAX_IMAGE_EDGE_PX, +): Dimensions { + const longest = Math.max(width, height); + if (longest <= maxEdge) return { width, height }; + const scale = maxEdge / longest; + // Rounding can never exceed the cap: each scaled edge is ≤ maxEdge before + // rounding, and the only values that round up land exactly ON the cap. + return { width: Math.round(width * scale), height: Math.round(height * scale) }; +} + +/** + * The smaller-wins rule: the converted JPEG is used only when it is actually + * FEWER bytes than the original (equal or larger → keep the original). + * Compared on raw byte counts (File/Blob `size`); base64 inflation affects + * both sides equally so the ratio is preserved. + */ +export function useConvertedImage(originalBytes: number, convertedBytes: number): boolean { + return convertedBytes < originalBytes; +} diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index 3a47219..9568813 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -4,6 +4,7 @@ import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import type { RenderedChunk } from "../../core/chunks"; import type { TurnMetricsEntry } from "../../core/metrics"; +import type { ConvertedJpeg } from "./adapter/image-encode"; import ChatView from "./ui/ChatView.svelte"; import Composer from "./ui/Composer.svelte"; import ModelSelector from "./ui/ModelSelector.svelte"; @@ -989,6 +990,127 @@ describe("Composer", () => { }); }); +describe("Composer image compression", () => { + /** Stage a PNG through the file picker and wait for the thumbnail. */ + async function stagePng(container: HTMLElement, content = "PNGDATA"): Promise { + const file = new File([content], "shot.png", { type: "image/png" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + } + + /** A fake converted JPEG (the outermost-edge stub — no canvas involved). */ + function fakeJpeg(): ConvertedJpeg { + return { blob: new Blob(["SMALLJPEG"], { type: "image/jpeg" }), mimeType: "image/jpeg" }; + } + + it("shows the compress toggle CHECKED by default for a staged PNG", async () => { + const { container } = render(Composer, { + props: { onSend: vi.fn(), encodeJpeg: vi.fn(async () => null) }, + }); + await stagePng(container); + + const toggle = screen.getByRole("checkbox", { name: "Compress image" }); + expect(toggle).toBeChecked(); + }); + + it("hides the compress toggle for already-lossy formats (JPEG)", async () => { + const { container } = render(Composer, { + props: { onSend: vi.fn(), encodeJpeg: vi.fn(async () => null) }, + }); + const file = new File(["JPG"], "photo.jpg", { type: "image/jpeg" }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + Object.defineProperty(input, "files", { value: [file], writable: false }); + input.dispatchEvent(new Event("change", { bubbles: true })); + await vi.waitFor(() => { + expect(screen.getByRole("button", { name: "Remove image" })).toBeInTheDocument(); + }); + + expect(screen.queryByRole("checkbox", { name: "Compress image" })).not.toBeInTheDocument(); + }); + + it("sends the converted JPEG (image/jpeg) when compress is on, encoding exactly once", async () => { + const encodeJpeg = vi.fn(async () => fakeJpeg()); + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend, encodeJpeg } }); + await stagePng(container); + + // The async conversion lands and the staged URL reflects what will be sent. + await vi.waitFor(() => { + expect(encodeJpeg).toHaveBeenCalledTimes(1); + }); + await vi.waitFor(() => { + const thumb = container.querySelector("img[alt='image/jpeg']"); + expect(thumb).not.toBeNull(); + expect(thumb?.getAttribute("src")).toMatch(/^data:image\/jpeg;base64,/); + }); + + await user.click(screen.getByRole("button", { name: "Send" })); + expect(onSend).toHaveBeenCalledTimes(1); + const [, images] = onSend.mock.calls[0] ?? []; + expect(images).toHaveLength(1); + expect(images[0]?.mimeType).toBe("image/jpeg"); + expect(images[0]?.url).toMatch(/^data:image\/jpeg;base64,/); + }); + + it("sends the ORIGINAL untouched bytes when the toggle is unchecked", async () => { + const encodeJpeg = vi.fn(async () => fakeJpeg()); + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend, encodeJpeg } }); + await stagePng(container); + await vi.waitFor(() => expect(encodeJpeg).toHaveBeenCalledTimes(1)); + + await user.click(screen.getByRole("checkbox", { name: "Compress image" })); + expect(screen.getByRole("checkbox", { name: "Compress image" })).not.toBeChecked(); + + await user.click(screen.getByRole("button", { name: "Send" })); + expect(onSend).toHaveBeenCalledTimes(1); + const [, images] = onSend.mock.calls[0] ?? []; + expect(images[0]?.mimeType).toBe("image/png"); + expect(images[0]?.url).toMatch(/^data:image\/png;base64,/); + }); + + it("re-checking uses the CACHED conversion — no second encode", async () => { + const encodeJpeg = vi.fn(async () => fakeJpeg()); + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend, encodeJpeg } }); + await stagePng(container); + await vi.waitFor(() => expect(encodeJpeg).toHaveBeenCalledTimes(1)); + + const toggle = screen.getByRole("checkbox", { name: "Compress image" }); + await user.click(toggle); // OFF → original + await user.click(toggle); // ON again → must reuse the cached JPEG + + await user.click(screen.getByRole("button", { name: "Send" })); + expect(encodeJpeg).toHaveBeenCalledTimes(1); // still exactly one encode + const [, images] = onSend.mock.calls[0] ?? []; + expect(images[0]?.mimeType).toBe("image/jpeg"); + expect(images[0]?.url).toMatch(/^data:image\/jpeg;base64,/); + }); + + it("falls back to the original image when conversion fails", async () => { + const encodeJpeg = vi.fn(async () => { + throw new Error("decode exploded"); + }); + const onSend = vi.fn(); + const user = userEvent.setup(); + const { container } = render(Composer, { props: { onSend, encodeJpeg } }); + await stagePng(container); + await vi.waitFor(() => expect(encodeJpeg).toHaveBeenCalledTimes(1)); + + await user.click(screen.getByRole("button", { name: "Send" })); + const [, images] = onSend.mock.calls[0] ?? []; + expect(images[0]?.mimeType).toBe("image/png"); + expect(images[0]?.url).toMatch(/^data:image\/png;base64,/); + }); +}); + describe("ModelSelector", () => { const optionValues = (el: HTMLElement): string[] => within(el) diff --git a/src/features/chat/ui/Composer.svelte b/src/features/chat/ui/Composer.svelte index 04c28cd..c7d9d02 100644 --- a/src/features/chat/ui/Composer.svelte +++ b/src/features/chat/ui/Composer.svelte @@ -1,5 +1,7 @@