From 66e5d3b105bfd2b34c6f35876bf33dbb3cb9dcae Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Tue, 2 Jun 2026 22:50:11 +0900 Subject: feat(chat): paste-to-attach images/PDFs with model capability check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add multimodal image/PDF input to the chat box via clipboard paste, gated by a graceful per-model capability check. UX: a pasted image/PDF inserts an inline token (【image:…】 / 【pdf:…】) into the draft, so attachments have ORDER relative to typed text and can be referenced positionally. The token is the only handle — deleting it (atomic Backspace/ Delete, or selection overlap) detaches the file; an input-reconciliation safety net detaches any attachment whose token is no longer intact. No preview strip. Capability check: resolveModelCapabilities reads models.dev modalities.input (new GET /models/capabilities, mirrors /context-limit). The input blocks Send (no tokens spent) only on a definitive 'no'; unknown capability (catalog offline / unmapped provider) stays permissive. Attachments require a fresh turn — Send is blocked while generating and /chat rejects content mid-turn (409). Attachments are EPHEMERAL: forwarded to the model for the turn via ordered AI SDK ImagePart/FilePart content, but never persisted (history keeps the text with [image]/[pdf] markers). Text-only turns serialize byte-identically to before. Limits (Anthropic-aligned, enforced at paste + re-validated server-side): PNG/JPEG/WebP/GIF/PDF; image ≤5MB, PDF ≤32MB, ≤20 attachments, ≤32MB total. core: UserContentPart types, models/attachments validator, capability resolver, agent.run+toModelMessages thread ordered content. api: /chat content validation + passthrough. frontend: attachment-tokens helper, ChatInput paste/token/gating, per-tab staged attachments, App.svelte capability fetch. +44 tests. --- packages/core/src/models/index.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'packages/core/src/models/index.ts') diff --git a/packages/core/src/models/index.ts b/packages/core/src/models/index.ts index 2fcd657..15d1ee2 100644 --- a/packages/core/src/models/index.ts +++ b/packages/core/src/models/index.ts @@ -1,5 +1,24 @@ +export { + ACCEPTED_ATTACHMENT_MEDIA_TYPES, + ACCEPTED_IMAGE_MEDIA_TYPES, + ACCEPTED_PDF_MEDIA_TYPE, + type AttachmentValidationError, + type AttachmentValidationResult, + base64ByteLength, + hasAttachments, + isAcceptedAttachmentMediaType, + isImageMediaType, + isPdfMediaType, + MAX_ATTACHMENTS, + MAX_IMAGE_BYTES, + MAX_PDF_BYTES, + MAX_TOTAL_ATTACHMENT_BYTES, + validateUserContent, +} from "./attachments.js"; export { getModelsCatalog, + type ModelInputCapabilities, resolveContextLimit, + resolveModelCapabilities, } from "./catalog.js"; export { ModelRegistry } from "./registry.js"; -- cgit v1.2.3