From fa0bd9c0e433b1abddc814b48a358c94954c7d36 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sat, 27 Jun 2026 19:15:04 +0900 Subject: feat(vision): consult_vision tool + vision settings API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend vision update (additive to wire@0.12.0 / transport-contract@0.22.0, no version bump). Contracts mirrored (.dispatch/transport-contract.reference.md): - VisionSettingsResponse + SetVisionSettingsRequest (GET/PUT /settings/vision). - Delta note: read_image -> consult_vision; numbered placeholders; compaction. Tool rendering (ChatView): - read_image test -> consult_vision (rendering is generic by toolName). - +2 tests: numbered-placeholder text chunk + [Compacted image] text chunk (both regular text chunks, render as-is — no special handling). New vision feature library (src/features/vision/): - logic/view-model.ts (32 tests): VisionSettings/VisionSettingsPatch types (consumer-defines-port), LoadVisionSettings/SaveVisionSettings ports + results, normalizeVisionSettings (network-seam coercion), parseImageLimit/ imageLimitChanged, compactionModelOptions (vision-capable models via chat's public isVisionModel + Auto sentinel), round-trip helpers, imageLimitLabel. - ui/VisionSettingsView.svelte (9 tests): imageLimit input + Save, compactionModel dropdown (Auto + vision-capable models), load-on-mount, save-on-change, error/saved feedback. - index.ts. Cross-unit seam: isVisionModel added to features/chat public index.ts (additive); imported through the public surface, not internals. Store wiring (src/app/store.svelte.ts): - visionSettings state + refreshVisionSettings (GET /settings/vision, normalized at the seam) + setVisionSettings (PUT, partial, returns merged) + VisionSettingsResult; seeded on boot; exposed on AppStore. +4 store tests. Mounted in App.svelte: new "Vision" sidebar view kind + VisionSettingsView in viewContent (not conversation-scoped); load/save adapters; visionManifest. Verification: svelte-check 0/0; vitest 948/948 (run twice, +47 since the prior vision commit); biome clean; vite build OK. See backend-handoff.md §2j. Not merged or pushed. --- src/features/vision/index.ts | 30 +++ src/features/vision/logic/view-model.test.ts | 198 ++++++++++++++++++ src/features/vision/logic/view-model.ts | 189 +++++++++++++++++ src/features/vision/ui/VisionSettingsView.svelte | 192 +++++++++++++++++ src/features/vision/ui/VisionSettingsView.test.ts | 241 ++++++++++++++++++++++ 5 files changed, 850 insertions(+) create mode 100644 src/features/vision/index.ts create mode 100644 src/features/vision/logic/view-model.test.ts create mode 100644 src/features/vision/logic/view-model.ts create mode 100644 src/features/vision/ui/VisionSettingsView.svelte create mode 100644 src/features/vision/ui/VisionSettingsView.test.ts (limited to 'src/features/vision') diff --git a/src/features/vision/index.ts b/src/features/vision/index.ts new file mode 100644 index 0000000..6956d75 --- /dev/null +++ b/src/features/vision/index.ts @@ -0,0 +1,30 @@ +export type { + CompactionModelOption, + ImageLimitParse, + LoadVisionSettings, + LoadVisionSettingsResult, + SaveVisionSettings, + SaveVisionSettingsResult, + VisionSettings, + VisionSettingsPatch, +} from "./logic/view-model"; +export { + AUTO_COMPACTION_MODEL, + compactionModelChanged, + compactionModelFromValue, + compactionModelOptions, + DEFAULT_IMAGE_LIMIT, + imageLimitChanged, + imageLimitLabel, + MAX_IMAGE_LIMIT, + normalizeVisionSettings, + parseImageLimit, + selectedCompactionValue, +} from "./logic/view-model"; +export { default as VisionSettingsView } from "./ui/VisionSettingsView.svelte"; + +/** Public module manifest — aggregated by the shell's "Loaded Modules" view. */ +export const manifest = { + name: "vision", + description: "Global vision settings (image compaction limit + model)", +} as const; diff --git a/src/features/vision/logic/view-model.test.ts b/src/features/vision/logic/view-model.test.ts new file mode 100644 index 0000000..b1db822 --- /dev/null +++ b/src/features/vision/logic/view-model.test.ts @@ -0,0 +1,198 @@ +import type { ModelMetadata } from "@dispatch/transport-contract"; +import { describe, expect, it } from "vitest"; +import { + AUTO_COMPACTION_MODEL, + compactionModelChanged, + compactionModelFromValue, + compactionModelOptions, + DEFAULT_IMAGE_LIMIT, + imageLimitChanged, + imageLimitLabel, + MAX_IMAGE_LIMIT, + normalizeVisionSettings, + parseImageLimit, + selectedCompactionValue, +} from "./view-model"; + +describe("normalizeVisionSettings", () => { + it("returns the value as-is for a well-formed body", () => { + expect(normalizeVisionSettings({ imageLimit: 5, compactionModel: "kimi/k2" })).toEqual({ + imageLimit: 5, + compactionModel: "kimi/k2", + }); + }); + + it("coerces a null compactionModel to null", () => { + expect(normalizeVisionSettings({ imageLimit: 10, compactionModel: null })).toEqual({ + imageLimit: 10, + compactionModel: null, + }); + }); + + it("defaults imageLimit when absent or non-numeric", () => { + expect(normalizeVisionSettings({ compactionModel: "kimi/k2" })).toEqual({ + imageLimit: DEFAULT_IMAGE_LIMIT, + compactionModel: "kimi/k2", + }); + expect(normalizeVisionSettings({ imageLimit: "oops", compactionModel: null })).toEqual({ + imageLimit: DEFAULT_IMAGE_LIMIT, + compactionModel: null, + }); + }); + + it("floors a fractional imageLimit", () => { + expect(normalizeVisionSettings({ imageLimit: 7.9, compactionModel: null }).imageLimit).toBe(7); + }); + + it("defaults for a null/non-object body (never crashes)", () => { + expect(normalizeVisionSettings(null)).toEqual({ + imageLimit: DEFAULT_IMAGE_LIMIT, + compactionModel: null, + }); + expect(normalizeVisionSettings("nope")).toEqual({ + imageLimit: DEFAULT_IMAGE_LIMIT, + compactionModel: null, + }); + }); + + it("treats a negative imageLimit as the default", () => { + expect(normalizeVisionSettings({ imageLimit: -3, compactionModel: null }).imageLimit).toBe( + DEFAULT_IMAGE_LIMIT, + ); + }); + + it("treats an empty compactionModel string as null", () => { + expect( + normalizeVisionSettings({ imageLimit: 10, compactionModel: "" }).compactionModel, + ).toBeNull(); + }); +}); + +describe("parseImageLimit", () => { + it("parses a non-negative integer", () => { + expect(parseImageLimit("5")).toEqual({ ok: true, value: 5 }); + expect(parseImageLimit("0")).toEqual({ ok: true, value: 0 }); + }); + + it("floors a fractional value", () => { + expect(parseImageLimit("7.9")).toEqual({ ok: true, value: 7 }); + }); + + it("trims whitespace", () => { + expect(parseImageLimit(" 12 ")).toEqual({ ok: true, value: 12 }); + }); + + it("errors on empty input", () => { + expect(parseImageLimit("")).toEqual({ ok: false, error: "Enter a number." }); + }); + + it("errors on non-numeric input", () => { + expect(parseImageLimit("lots").ok).toBe(false); + }); + + it("errors on a negative value", () => { + expect(parseImageLimit("-1").ok).toBe(false); + }); + + it("errors when above the max", () => { + expect(parseImageLimit(String(MAX_IMAGE_LIMIT + 1)).ok).toBe(false); + }); + + it("accepts the max", () => { + expect(parseImageLimit(String(MAX_IMAGE_LIMIT))).toEqual({ ok: true, value: MAX_IMAGE_LIMIT }); + }); +}); + +describe("imageLimitChanged", () => { + it("is true when the typed value differs after normalization", () => { + expect(imageLimitChanged("5", 10)).toBe(true); + }); + + it("is false when equal", () => { + expect(imageLimitChanged("10", 10)).toBe(false); + }); + + it("is false for invalid input", () => { + expect(imageLimitChanged("abc", 10)).toBe(false); + expect(imageLimitChanged("", 10)).toBe(false); + }); + + it("compares after flooring", () => { + expect(imageLimitChanged("7.9", 7)).toBe(false); + }); +}); + +describe("compactionModelOptions", () => { + const modelInfo: Record = { + "kimi/k2": { vision: true }, + "kimi/k1.5": { vision: true }, + "umans/glm-5.2": { vision: false }, + "openai/gpt-4": { contextWindow: 128000 }, + }; + + it("includes the Auto option first", () => { + const opts = compactionModelOptions([], {}); + expect(opts).toHaveLength(1); + expect(opts[0]?.auto).toBe(true); + expect(opts[0]?.value).toBe(AUTO_COMPACTION_MODEL); + }); + + it("includes only vision-capable models, in catalog order", () => { + const models = ["umans/glm-5.2", "kimi/k2", "openai/gpt-4", "kimi/k1.5"]; + const opts = compactionModelOptions(models, modelInfo); + expect(opts.map((o) => o.label)).toEqual(["Auto (server-selected)", "kimi/k2", "kimi/k1.5"]); + }); + + it("excludes non-vision models even with metadata present", () => { + const opts = compactionModelOptions(["umans/glm-5.2"], modelInfo); + expect(opts).toHaveLength(1); // only Auto + }); +}); + +describe("compactionModel value round-trip", () => { + it("selectedCompactionValue maps null to the auto sentinel", () => { + expect(selectedCompactionValue(null)).toBe(AUTO_COMPACTION_MODEL); + }); + + it("selectedCompactionValue maps a model name to itself", () => { + expect(selectedCompactionValue("kimi/k2")).toBe("kimi/k2"); + }); + + it("compactionModelFromValue maps the auto sentinel back to null", () => { + expect(compactionModelFromValue(AUTO_COMPACTION_MODEL)).toBeNull(); + }); + + it("compactionModelFromValue maps a model name to itself", () => { + expect(compactionModelFromValue("kimi/k2")).toBe("kimi/k2"); + }); +}); + +describe("compactionModelChanged", () => { + it("is true when the value maps to a different model", () => { + expect(compactionModelChanged("kimi/k2", null)).toBe(true); + expect(compactionModelChanged(AUTO_COMPACTION_MODEL, "kimi/k2")).toBe(true); + }); + + it("is false when equal (null vs auto, or same model)", () => { + expect(compactionModelChanged(AUTO_COMPACTION_MODEL, null)).toBe(false); + expect(compactionModelChanged("kimi/k2", "kimi/k2")).toBe(false); + }); +}); + +describe("imageLimitLabel", () => { + it("labels null as loading", () => { + expect(imageLimitLabel(null)).toBe("Loading…"); + }); + + it("labels 0 as disabled", () => { + expect(imageLimitLabel(0)).toBe("0 (compaction disabled)"); + }); + + it("labels the default with (default)", () => { + expect(imageLimitLabel(DEFAULT_IMAGE_LIMIT)).toBe(`${DEFAULT_IMAGE_LIMIT} (default)`); + }); + + it("labels other values plainly", () => { + expect(imageLimitLabel(7)).toBe("7"); + }); +}); diff --git a/src/features/vision/logic/view-model.ts b/src/features/vision/logic/view-model.ts new file mode 100644 index 0000000..97a7093 --- /dev/null +++ b/src/features/vision/logic/view-model.ts @@ -0,0 +1,189 @@ +import type { ModelMetadata } from "@dispatch/transport-contract"; +import { isVisionModel } from "../../chat"; + +/** + * Pure core for the vision settings feature — zero DOM, zero effects, zero Svelte. + * + * The global vision configuration (`GET`/`PUT /settings/vision`) controls image + * compaction: how many native images a vision model keeps per turn before the + * oldest are transcribed to text (`imageLimit`), and which vision model does the + * transcribing (`compactionModel`, null = auto). This module is the view-model + * seam: typed settings, parse/validate, dirty-check, network normalization, and + * the vision-capable model option list. The composition root adapts the store's + * HTTP calls to the injected ports. + */ + +// ── Types (owned locally — consumer-defines-port; the contract shapes are a +// plain REST surface not in a shared contract package version bump, mirroring +// the heartbeat/mcp pattern). If the backend promotes these to a shared +// package, swap the local types for the imports + re-mirror. ────────────── + +/** The global vision settings (mirrors `VisionSettingsResponse`). */ +export interface VisionSettings { + /** Max native images per turn (default 10); 0 disables compaction. */ + readonly imageLimit: number; + /** Which model transcribes old images (`/`), or null = auto. */ + readonly compactionModel: string | null; +} + +/** A partial update (mirrors `SetVisionSettingsRequest`). */ +export interface VisionSettingsPatch { + readonly imageLimit?: number; + readonly compactionModel?: string | null; +} + +// ── Injected ports (consumer-defines-port). ─────────────────────────────────── + +/** Outcome of loading the vision settings. */ +export type LoadVisionSettingsResult = + | { readonly ok: true; readonly settings: VisionSettings } + | { readonly ok: false; readonly error: string }; + +/** Outcome of saving a partial vision-settings update. */ +export type SaveVisionSettingsResult = + | { readonly ok: true; readonly settings: VisionSettings } + | { readonly ok: false; readonly error: string }; + +export type LoadVisionSettings = () => Promise; +export type SaveVisionSettings = (patch: VisionSettingsPatch) => Promise; + +// ── Constants ──────────────────────────────────────────────────────────────── + +/** The backend's default image limit when none is persisted. */ +export const DEFAULT_IMAGE_LIMIT = 10; +/** Upper bound for the image limit input (defensive — the backend owns real clamping). */ +export const MAX_IMAGE_LIMIT = 1000; + +// ── Network normalization (pure; coerces untyped JSON at the seam). ────────── + +/** + * Coerce an untyped JSON body (from `GET /settings/vision`) into a valid + * `VisionSettings`. A malformed/partial response can never crash the renderer: + * `imageLimit` falls back to the default; `compactionModel` coerces to null. + */ +export function normalizeVisionSettings(body: unknown): VisionSettings { + if (body === null || typeof body !== "object") { + return { imageLimit: DEFAULT_IMAGE_LIMIT, compactionModel: null }; + } + const raw = body as Record; + const limitRaw = raw.imageLimit; + const imageLimit = + typeof limitRaw === "number" && Number.isFinite(limitRaw) && limitRaw >= 0 + ? Math.floor(limitRaw) + : DEFAULT_IMAGE_LIMIT; + const modelRaw = raw.compactionModel; + const compactionModel = typeof modelRaw === "string" && modelRaw.length > 0 ? modelRaw : null; + return { imageLimit, compactionModel }; +} + +// ── imageLimit parse / validate ─────────────────────────────────────────────── + +/** Result of parsing a typed image-limit string. */ +export type ImageLimitParse = + | { readonly ok: true; readonly value: number } + | { readonly ok: false; readonly error: string }; + +/** + * Parse a typed image-limit string into a non-negative integer (floored, + * clamped to [0, MAX_IMAGE_LIMIT]). Empty or non-numeric input is an ERROR + * (so the UI can disable submit + message), NOT a silent default. + */ +export function parseImageLimit(raw: string): ImageLimitParse { + const trimmed = raw.trim(); + if (trimmed.length === 0) { + return { ok: false, error: "Enter a number." }; + } + const n = Number(trimmed); + if (!Number.isFinite(n) || n < 0) { + return { ok: false, error: "Must be 0 or a positive number." }; + } + const floored = Math.floor(n); + if (floored > MAX_IMAGE_LIMIT) { + return { ok: false, error: `Must be at most ${MAX_IMAGE_LIMIT}.` }; + } + return { ok: true, value: floored }; +} + +/** + * Whether saving `typed` would change the `current` image limit. A no-op save + * (empty/invalid, or equal) should be disabled. This is the dirty-check for the + * input — it must NOT mutate or clamp, only compare. + */ +export function imageLimitChanged(typed: string, current: number): boolean { + const parsed = parseImageLimit(typed); + if (!parsed.ok) return false; + return parsed.value !== current; +} + +// ── compactionModel option list ─────────────────────────────────────────────── + +/** + * The sentinel value for the "Auto" option (null compactionModel — the server + * auto-selects a vision model). Used as the `