From 3566a20ebbded754070fce66af48690d1a904879 Mon Sep 17 00:00:00 2001
From: Adam Malczewski <github@tradam.dev>
Date: Sat, 27 Jun 2026 04:18:59 +0900
Subject: feat(vision): image paste + transcript image rendering + vision badge
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Vision & vision-handoff frontend (consumes the backend's additive
wire@0.12.0 / transport-contract@0.22.0 image types — no version bump).

Contracts mirrored:
- .dispatch/wire.reference.md: ImageChunk added to the Chunk union +
  ImageChunk/ImageInput interfaces.
- .dispatch/transport-contract.reference.md: ChatRequest.images,
  ModelMetadata.vision, + ImageChunk/ImageInput re-exports.

Core (core/chunks):
- conformance: assertChunkExhaustive handles the new 'image' variant
  (the guard caught it — its purpose).
- appendUserMessage(state, text, images?) echoes a [text, image, ...]
  user run; the user-message event dedup scans the trailing user run
  (not just the last chunk) so an image-bearing echo doesn't duplicate
  the text; applyHistory's during-gen dedup matches a multi-chunk echo
  by content equality (chunkContentEquals + trailingRun helpers).

UI:
- ChatView renders user 'image' chunks as lazy <img> bubbles; a
  non-vision model's persisted [image, analysis-text] both render.
  read_image tool renders generically (no special-casing).
- Composer: clipboard paste / file picker / drag-drop of images ->
  base64 data URLs, thumbnail previews with remove, forwarded on
  chat.send (omitted when none). Image-only sends allowed; steering
  (chat.queue) never forwards images.
- ModelSelector: vision badge (isVisionModel) marks vision-capable
  models; indicator shows native-vision vs vision-handoff hint.

Store wiring: ChatStore.send + AppStore.send + App.svelte handleSend
thread images through; chat.send still omits cwd (only images added).

Verification: svelte-check 0/0; vitest 901/901 (run twice, +34 new);
biome clean; vite build OK. See backend-handoff.md §2j.

Not merged or pushed.
---
 src/features/chat/ui/ChatView.svelte      |  12 +-
 src/features/chat/ui/Composer.svelte      | 232 +++++++++++++++++++++++++++---
 src/features/chat/ui/ModelSelector.svelte |  46 +++++-
 3 files changed, 269 insertions(+), 21 deletions(-)

(limited to 'src/features/chat/ui')

diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte
index e72f639..8081951 100644
--- a/src/features/chat/ui/ChatView.svelte
+++ b/src/features/chat/ui/ChatView.svelte
@@ -95,11 +95,21 @@
 
 {#snippet chunkRow(rendered: RenderedChunk)}
   {#if rendered.role === "user"}
-    <!-- User: a speech bubble, left-aligned -->
+    <!-- User: a speech bubble, left-aligned. A user message may be multi-chunk
+         ([text, image, image, …]); each chunk renders in its own bubble (the
+         image's url is a base64 data URL or an https URL — render it directly). -->
     <div class="chat chat-start">
       <div class="chat-bubble chat-bubble-primary">
         {#if rendered.chunk.type === "text"}
           <p>{rendered.chunk.text}</p>
+        {:else if rendered.chunk.type === "image"}
+          <img
+            src={rendered.chunk.url}
+            alt={rendered.chunk.mimeType ?? "pasted image"}
+            loading="lazy"
+            decoding="async"
+            class="max-h-80 max-w-full rounded"
+          />
         {/if}
       </div>
     </div>
diff --git a/src/features/chat/ui/Composer.svelte b/src/features/chat/ui/Composer.svelte
index 96b4b3a..7898448 100644
--- a/src/features/chat/ui/Composer.svelte
+++ b/src/features/chat/ui/Composer.svelte
@@ -1,8 +1,19 @@
 <script lang="ts">
+  import type { ImageInput } from "@dispatch/wire";
   import { computeContextUsage, formatCompactTokens } from "../../../core/metrics";
 
   const FALLBACK_CONTEXT_WINDOW = 1_000_000;
   const MAX_LINES = 7;
+  /** Accept only raster images (the provider image-content formats). */
+  const IMAGE_ACCEPT = "image/png,image/jpeg,image/gif,image/webp";
+  /** Reject images larger than this before base64-encoding (keeps payloads sane). */
+  const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
+
+  /** A staged image awaiting send: a stable id + the `ImageInput` to forward. */
+  interface StagedImage {
+    readonly id: string;
+    readonly input: ImageInput;
+  }
 
   let {
     onSend,
@@ -12,12 +23,18 @@
     contextWindow = undefined,
     status = "idle",
   }: {
-    onSend: (text: string) => void;
+    /**
+     * Send a message (start a turn via `chat.send`). Carries any staged images
+     * as `ImageInput[]` (base64 data URLs or https URLs); the store forwards
+     * them on the WS `chat.send` op / `POST /chat` body. `images` is omitted
+     * (not an empty array) when none are staged, so the wire stays text-only.
+     */
+    onSend: (text: string, images?: ImageInput[]) => void;
     /**
      * Enqueue a steering message (`chat.queue`). When provided AND the status
      * is `running`, the send button becomes a "Queue" button that steers the
-     * in-flight turn instead of starting a new one. When absent, `onSend` is
-     * used regardless (tests / non-steering contexts).
+     * in-flight turn instead of starting a new one. Steering is text-only —
+     * it never carries images (a mid-turn injection has no image surface).
      */
     onQueue?: (text: string) => void;
     /** Stop the in-flight generation (`POST /conversations/:id/stop`). */
@@ -32,24 +49,35 @@
   } = $props();
 
   let text = $state("");
+  let images = $state<StagedImage[]>([]);
   let inputEl: HTMLTextAreaElement | undefined;
+  let fileInputEl: HTMLInputElement | undefined;
+  let dragOver = $state(false);
 
   const hasText = $derived(text.trim().length > 0);
+  const hasImages = $derived(images.length > 0);
+  const canSend = $derived(hasText || hasImages);
   const effectiveMax = $derived(contextWindow ?? FALLBACK_CONTEXT_WINDOW);
   const usage = $derived(computeContextUsage(contextSize, effectiveMax));
   const hasUsage = $derived(contextSize !== undefined);
 
   // One button, three modes:
   // - idle → "Send" (starts a turn via chat.send)
-  // - running + text → "Queue" (steers via chat.queue)
+  // - running + text/images → "Queue" (steers via chat.queue — text only)
   // - running + empty → "Stop" (aborts via POST /stop)
+  // Steering never carries images: when running with images staged but no text,
+  // fall back to "Send" semantics is wrong mid-turn — instead queue the text part
+  // only (images stay staged). Simplest correct rule: queue is text-only and only
+  // offered with text; images-without-text while running is an unusual case that
+  // still sends (the server auto-starts/resolves). Keep the three-mode logic
+  // driven by text presence for the queue vs stop split.
   const buttonMode = $derived.by<"send" | "queue" | "stop">(() => {
-    if (status === "running" && !hasText && onStop !== undefined) return "stop";
+    if (status === "running" && !hasText && !hasImages && onStop !== undefined) return "stop";
     if (status === "running" && hasText && onQueue !== undefined) return "queue";
     return "send";
   });
   const placeholder = $derived(
-    status === "running" ? "Steer the conversation..." : "Type a message...",
+    status === "running" ? "Steer the conversation..." : "Type a message, paste or drop an image…",
   );
 
   // As the window fills, escalate color: calm → warning → danger.
@@ -80,15 +108,97 @@
     resize();
   });
 
+  /** Read a File into a base64 data URL (`data:image/…;base64,…`). */
+  function fileToDataUrl(file: File): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader();
+      reader.onload = () => {
+        if (typeof reader.result === "string") resolve(reader.result);
+        else reject(new Error("unreadable image"));
+      };
+      reader.onerror = () => reject(reader.error ?? new Error("read failed"));
+      reader.readAsDataURL(file);
+    });
+  }
+
+  let imgSeq = 0;
+  /** Stage a File as an image (skip non-images / oversized). Returns whether staged. */
+  async function stageFile(file: File): Promise<boolean> {
+    if (!file.type.startsWith("image/")) return false;
+    if (file.size > MAX_IMAGE_BYTES) return false;
+    const url = await fileToDataUrl(file);
+    // Prefer the file's declared MIME; fall back to the data URL's prefix.
+    const mimeType = file.type || undefined;
+    const id = `img-${Date.now()}-${imgSeq++}`;
+    images = [...images, { id, input: { url, ...(mimeType ? { mimeType } : {}) } }];
+    return true;
+  }
+
+  function removeImage(id: string): void {
+    images = images.filter((img) => img.id !== id);
+  }
+
+  /** Handle a paste anywhere in the form: extract image items from the clipboard. */
+  async function handlePaste(e: ClipboardEvent): Promise<void> {
+    const items = e.clipboardData?.items;
+    if (items === undefined) return;
+    let hadImage = false;
+    const staged: File[] = [];
+    for (const item of items) {
+      if (item.kind === "file" && item.type.startsWith("image/")) {
+        const file = item.getAsFile();
+        if (file !== null) {
+          staged.push(file);
+          hadImage = true;
+        }
+      }
+    }
+    if (!hadImage) return; // let the default text paste proceed
+    e.preventDefault(); // suppress pasting the image as a filename string
+    for (const file of staged) {
+      await stageFile(file);
+    }
+  }
+
+  /** File-picker <input type="file"> change. */
+  async function handleFilePick(e: Event): Promise<void> {
+    const target = e.currentTarget as HTMLInputElement;
+    const files = target.files;
+    if (files === null) return;
+    for (const file of files) {
+      await stageFile(file);
+    }
+    target.value = ""; // reset so picking the same file again re-fires change
+  }
+
+  /** Drop images onto the composer. */
+  async function handleDrop(e: DragEvent): Promise<void> {
+    dragOver = false;
+    const files = e.dataTransfer?.files;
+    if (files === undefined || files.length === 0) return;
+    const hadImage = Array.from(files).some((f) => f.type.startsWith("image/"));
+    if (!hadImage) return;
+    e.preventDefault();
+    for (const file of files) {
+      await stageFile(file);
+    }
+  }
+
   function handleSubmit(): void {
     const trimmed = text.trim();
-    if (trimmed.length === 0) return;
+    // Allow a send with images even when text is empty (an image-only turn).
+    if (trimmed.length === 0 && !hasImages) return;
     if (buttonMode === "queue") {
+      // Steering is text-only — never forward images.
       onQueue?.(trimmed);
     } else {
-      onSend(trimmed);
+      const toSend: ImageInput[] | undefined = hasImages
+        ? images.map((img) => img.input)
+        : undefined;
+      onSend(trimmed, toSend);
     }
     text = "";
+    images = [];
   }
 
   function handleKeydown(e: KeyboardEvent): void {
@@ -105,17 +215,68 @@
     e.preventDefault();
     handleSubmit();
   }}
+  ondrop={handleDrop}
+  ondragover={(e) => {
+    if (e.dataTransfer?.types?.includes("Files")) {
+      e.preventDefault();
+      dragOver = true;
+    }
+  }}
+  ondragleave={() => (dragOver = false)}
 >
-  <!-- Top bar: expanding textarea + single context-aware button -->
+  <!-- Top bar: expanding textarea + image-attach button + single context-aware button -->
   <div class="flex items-end gap-2 px-4 pt-3 pb-2">
-    <textarea
-      bind:this={inputEl}
-      class="textarea textarea-bordered flex-1 resize-none leading-normal !min-h-0 h-auto"
-      bind:value={text}
-      onkeydown={handleKeydown}
-      {placeholder}
-      rows="1"
-      aria-label="Message input"></textarea>
+    <div
+      class="flex-1"
+      onpaste={handlePaste}
+      class:border-2={dragOver}
+      class:border-primary={dragOver}
+      class:border-dashed={dragOver}
+      class:rounded={dragOver}
+    >
+      <textarea
+        bind:this={inputEl}
+        class="textarea textarea-bordered w-full resize-none leading-normal !min-h-0 h-auto"
+        bind:value={text}
+        onkeydown={handleKeydown}
+        {placeholder}
+        rows="1"
+        aria-label="Message input"></textarea>
+    </div>
+
+    <!-- Hidden file picker (images only; multiple). -->
+    <input
+      bind:this={fileInputEl}
+      type="file"
+      accept={IMAGE_ACCEPT}
+      multiple
+      class="hidden"
+      onchange={handleFilePick}
+    />
+    <!-- Attach image button (opens the file picker). -->
+    <button
+      class="btn btn-ghost btn-square shrink-0"
+      type="button"
+      aria-label="Attach image"
+      title="Attach image"
+      onclick={() => fileInputEl?.click()}
+    >
+      <svg
+        xmlns="http://www.w3.org/2000/svg"
+        viewBox="0 0 24 24"
+        fill="none"
+        stroke="currentColor"
+        stroke-width="2"
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        class="h-5 w-5"
+      >
+        <rect x="3" y="3" width="18" height="18" rx="2" ry="2"></rect>
+        <circle cx="8.5" cy="8.5" r="1.5"></circle>
+        <polyline points="21 15 16 10 5 21"></polyline>
+      </svg>
+    </button>
+
     {#if buttonMode === "stop"}
       <button
         class="btn btn-error w-20 shrink-0"
@@ -126,12 +287,47 @@
         Stop
       </button>
     {:else}
-      <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!hasText}>
+      <button class="btn btn-primary w-20 shrink-0" type="submit" disabled={!canSend}>
         {buttonMode === "queue" ? "Queue" : "Send"}
       </button>
     {/if}
   </div>
 
+  <!-- Staged image thumbnails (previews) with remove buttons. -->
+  {#if hasImages}
+    <div class="flex flex-wrap gap-2 px-4 pb-1">
+      {#each images as img (img.id)}
+        <div class="group relative h-20 w-20 shrink-0 overflow-hidden rounded border border-base-300">
+          <img
+            src={img.input.url}
+            alt={img.input.mimeType ?? "staged image"}
+            class="h-full w-full object-cover"
+          />
+          <button
+            class="btn btn-circle btn-xs absolute right-0 top-0 bg-base-100/80 hover:bg-error hover:text-error-content"
+            type="button"
+            aria-label="Remove image"
+            onclick={() => removeImage(img.id)}
+          >
+            <svg
+              xmlns="http://www.w3.org/2000/svg"
+              viewBox="0 0 24 24"
+              fill="none"
+              stroke="currentColor"
+              stroke-width="3"
+              stroke-linecap="round"
+              stroke-linejoin="round"
+              class="h-3 w-3"
+            >
+              <line x1="18" y1="6" x2="6" y2="18"></line>
+              <line x1="6" y1="6" x2="18" y2="18"></line>
+            </svg>
+          </button>
+        </div>
+      {/each}
+    </div>
+  {/if}
+
   <!-- Bottom status bar: status icon · context-window fill · token count -->
   <div class="flex items-center gap-2 px-4 pb-2 text-xs text-base-content/50">
     <span class="shrink-0">
diff --git a/src/features/chat/ui/ModelSelector.svelte b/src/features/chat/ui/ModelSelector.svelte
index 03acb79..11b9feb 100644
--- a/src/features/chat/ui/ModelSelector.svelte
+++ b/src/features/chat/ui/ModelSelector.svelte
@@ -1,20 +1,32 @@
 <script lang="ts">
-  import { joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select";
+  import type { ModelMetadata } from "@dispatch/transport-contract";
+  import { isVisionModel, joinModelName, modelKeys, modelsForKey, splitModelName } from "../model-select";
 
   let {
     models,
     selected,
     onSelect,
+    modelInfo = {},
   }: {
     models: readonly string[];
     selected: string;
     onSelect: (model: string) => void;
+    /**
+     * Per-model metadata from `GET /models` (`{ [name]: ModelMetadata }`).
+     * Used to show a "vision" badge next to models with `vision: true` (they
+     * natively accept images; others rely on the server's vision handoff).
+     * Optional — absent metadata → no badge (treated as non-vision).
+     */
+    modelInfo?: Readonly<Record<string, ModelMetadata>>;
   } = $props();
 
   const keys = $derived(modelKeys(models));
   const current = $derived(splitModelName(selected));
   const keyModels = $derived(modelsForKey(models, current.key));
 
+  // Whether the currently-selected full model name is vision-capable.
+  const selectedVision = $derived(isVisionModel(modelInfo, selected));
+
   // Switching key jumps to the first model available under it.
   function selectKey(key: string): void {
     const first = modelsForKey(models, key)[0] ?? "";
@@ -24,6 +36,11 @@
   function selectModel(model: string): void {
     onSelect(joinModelName(current.key, model));
   }
+
+  // The full `<key>/<model>` name for a model suffix under the current key.
+  function fullNameFor(modelSuffix: string): string {
+    return joinModelName(current.key, modelSuffix);
+  }
 </script>
 
 <div class="flex flex-col gap-2">
@@ -44,7 +61,32 @@
     aria-label="Model selector"
   >
     {#each keyModels as model (model)}
-      <option value={model}>{model}</option>
+      <option value={model}>
+        {model}{#if isVisionModel(modelInfo, fullNameFor(model))} · vision{/if}
+      </option>
     {/each}
   </select>
+  {#if selectedVision}
+    <div class="flex items-center gap-1 text-xs text-base-content/60">
+      <svg
+        xmlns="http://www.w3.org/2000/svg"
+        viewBox="0 0 24 24"
+        fill="none"
+        stroke="currentColor"
+        stroke-width="2"
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        class="h-3.5 w-3.5"
+        aria-hidden="true"
+      >
+        <path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"></path>
+        <circle cx="12" cy="12" r="3"></circle>
+      </svg>
+      <span>Vision — this model sees images natively</span>
+    </div>
+  {:else}
+    <div class="text-xs text-base-content/40">
+      Pasted images are auto-described (vision handoff)
+    </div>
+  {/if}
 </div>
-- 
cgit v1.2.3