From 66e5d3b105bfd2b34c6f35876bf33dbb3cb9dcae Mon Sep 17 00:00:00 2001
From: Adam Malczewski <github@tradam.dev>
Date: Tue, 2 Jun 2026 22:50:11 +0900
Subject: feat(chat): paste-to-attach images/PDFs with model capability check
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add multimodal image/PDF input to the chat box via clipboard paste, gated by a
graceful per-model capability check.

UX: a pasted image/PDF inserts an inline token (【image:…】 / 【pdf:…】) into the
draft, so attachments have ORDER relative to typed text and can be referenced
positionally. The token is the only handle — deleting it (atomic Backspace/
Delete, or selection overlap) detaches the file; an input-reconciliation safety
net detaches any attachment whose token is no longer intact. No preview strip.

Capability check: resolveModelCapabilities reads models.dev modalities.input
(new GET /models/capabilities, mirrors /context-limit). The input blocks Send
(no tokens spent) only on a definitive 'no'; unknown capability (catalog offline
/ unmapped provider) stays permissive. Attachments require a fresh turn — Send is
blocked while generating and /chat rejects content mid-turn (409).

Attachments are EPHEMERAL: forwarded to the model for the turn via ordered AI SDK
ImagePart/FilePart content, but never persisted (history keeps the text with
[image]/[pdf] markers). Text-only turns serialize byte-identically to before.

Limits (Anthropic-aligned, enforced at paste + re-validated server-side):
PNG/JPEG/WebP/GIF/PDF; image ≤5MB, PDF ≤32MB, ≤20 attachments, ≤32MB total.

core: UserContentPart types, models/attachments validator, capability resolver,
agent.run+toModelMessages thread ordered content. api: /chat content validation +
passthrough. frontend: attachment-tokens helper, ChatInput paste/token/gating,
per-tab staged attachments, App.svelte capability fetch. +44 tests.
---
 packages/core/src/models/catalog.ts | 50 +++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

(limited to 'packages/core/src/models/catalog.ts')

diff --git a/packages/core/src/models/catalog.ts b/packages/core/src/models/catalog.ts
index dea4647..ac310b1 100644
--- a/packages/core/src/models/catalog.ts
+++ b/packages/core/src/models/catalog.ts
@@ -18,6 +18,15 @@ interface ModelsDevModel {
 		context?: number;
 		output?: number;
 	};
+	/**
+	 * Input/output modalities the model accepts. We read `input` to decide
+	 * whether the model can take image / pdf attachments. Absent on older
+	 * catalog entries — treated as "unknown" (capability resolves to `null`).
+	 */
+	modalities?: {
+		input?: string[];
+		output?: string[];
+	};
 }
 
 interface ModelsDevProvider {
@@ -172,6 +181,47 @@ export async function resolveContextLimit(
 	return null;
 }
 
+/**
+ * Image / PDF input capabilities for a model, resolved from the models.dev
+ * catalog's `modalities.input` list.
+ */
+export interface ModelInputCapabilities {
+	/** Model accepts image input (vision). */
+	image: boolean;
+	/** Model accepts PDF/document input. */
+	pdf: boolean;
+}
+
+/**
+ * Resolve whether a model accepts image / pdf input for the given Dispatch
+ * provider + model id. Returns `null` when the capability is UNKNOWN — i.e. the
+ * provider is unsupported/unmapped, the model is absent from the catalog, the
+ * entry predates the `modalities` field, or the catalog is unavailable. Callers
+ * should treat `null` as "can't verify" (optimistic allow) rather than a
+ * definitive "no", so a temporary catalog outage never disables a known-good
+ * vision model.
+ *
+ * A non-null result means the catalog DID describe the model's input modalities
+ * — `{ image, pdf }` then reflects exactly what it advertises (a definitive
+ * yes/no for each).
+ */
+export async function resolveModelCapabilities(
+	provider: string,
+	modelId: string,
+): Promise<ModelInputCapabilities | null> {
+	const candidates = PROVIDER_MAP[provider];
+	if (!candidates || !modelId) return null;
+
+	const catalog = await getModelsCatalog();
+	for (const providerId of candidates) {
+		const input = catalog[providerId]?.models?.[modelId]?.modalities?.input;
+		if (Array.isArray(input)) {
+			return { image: input.includes("image"), pdf: input.includes("pdf") };
+		}
+	}
+	return null;
+}
+
 /** Test-only: reset the in-process memo so a test can re-exercise loading. */
 export function __resetCatalogCacheForTests(): void {
 	cached = null;
-- 
cgit v1.2.3