From d054dd8f28449653c1ef6e64c51eb475bb4d671d Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:07:08 +0200 Subject: [PATCH 1/2] feat(agent): region tools set and read every zoom, annotation and audio setting setZoom/addZoom take focusMode, hideCursor and customScale (written as the inspector writes a typed level, refused past the clip's zoomScaleLimit). add/setAnnotation take the type (text, arrow, privacy blur) through convertAnnotationKind, with figureData/blurData validated by the schema's own. add/setAudio take fades capped at the inspector's AUDIO_FADE_MAX_MS. getCurrentDocument reads all of it back. Part of #1180 --- electron/ai-edition/agent-tools.test.ts | 276 ++++++++++++ electron/ai-edition/agent-tools.ts | 392 ++++++++++++++++-- .../ai-edition/deep-agent/service.test.ts | 12 + electron/ai-edition/deep-agent/service.ts | 27 +- src/components/ai-edition/RightPanes.tsx | 15 +- .../ai-edition/v4/FloatingInspector.tsx | 11 +- src/lib/ai-edition/document/audioTracks.ts | 7 + src/lib/ai-edition/schema/index.ts | 4 +- src/lib/ai-edition/timeline/zoom-scale.ts | 12 + 9 files changed, 704 insertions(+), 52 deletions(-) diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 7ecf29988..4a9b07a6e 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -7,6 +7,7 @@ import { documentSchema, } from "../../src/lib/ai-edition/schema"; import { ZOOM_DEPTH_SCALES } from "../../src/lib/ai-edition/timeline/zoom-scale"; +import { zoomScaleLimit } from "../../src/native/sceneDescription"; import { executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES } from "./agent-tools"; import { measureTextInShippedFonts } from "./text-measure"; @@ -2507,6 +2508,34 @@ describe("addAudio / setAudio", () => { }); }); + it("fades a track in and out, reads the fades back, and refuses one past the inspector's cap", () => { + const placed = place(withAudioAsset(), { + assetId: "audio_1", + startSec: 2, + endSec: 6, + fadeInMs: 800, + }); + expect(placed.document?.audioTracks[0]).toMatchObject({ fadeInMs: 800, fadeOutMs: 0 }); + const id = JSON.parse(placed.resultJson).audioId; + const faded = executeAgentTool( + placed.document as AxcutDocument, + "setAudio", + JSON.stringify({ audioId: id, fadeOutMs: 1500 }), + ); + expect(JSON.parse(faded.resultJson)).toMatchObject({ fadeInMs: 800, fadeOutMs: 1500 }); + const snapshot = JSON.parse( + executeAgentTool(faded.document as AxcutDocument, "getCurrentDocument", "").resultJson, + ); + expect(snapshot.audioTracks[0]).toMatchObject({ fadeInMs: 800, fadeOutMs: 1500 }); + expect( + executeAgentTool( + placed.document as AxcutDocument, + "setAudio", + JSON.stringify({ audioId: id, fadeInMs: 5001 }), + ).ok, + ).toBe(false); + }); + it("setAudio applies the same offset guard as addAudio", () => { const placed = place(withAudioAsset(20), { assetId: "audio_1", startSec: 2, endSec: 6 }); const id = JSON.parse(placed.resultJson).audioId; @@ -3198,3 +3227,250 @@ describe("zoom transitions are reported, not hidden (#1028)", () => { ]); }); }); + +// #1180 batch C: what the inspector sets on a zoom or an annotation, the agent sets and reads back. +describe("region tools reach the inspector's settings (#1180)", () => { + const run = (doc: AxcutDocument, name: string, args: Record) => { + const result = executeAgentTool(doc, name, JSON.stringify(args)); + return { ...result, payload: JSON.parse(result.resultJson) }; + }; + const snapshot = (doc: AxcutDocument | undefined) => + JSON.parse(executeAgentTool(doc as AxcutDocument, "getCurrentDocument", "").resultJson); + + it("sets a zoom's focus mode, hidden cursor and free level, and reads them back", () => { + const added = run(fixtureDocument(), "addZoom", { + startSec: 2, + endSec: 4, + focusMode: "auto", + hideCursor: true, + customScale: 1.6, + }); + expect(added.ok).toBe(true); + expect(added.document?.zoomRanges[0]).toMatchObject({ + focusMode: "auto", + hideCursor: true, + customScale: 1.6, + }); + expect(added.payload).toMatchObject({ + focusMode: "auto", + hideCursor: true, + renderedScale: 1.6, + }); + expect(() => documentSchema.parse(added.document)).not.toThrow(); + expect(snapshot(added.document).zoomRanges[0]).toMatchObject({ + focusMode: "auto", + hideCursor: true, + customScale: 1.6, + renderedScale: 1.6, + }); + + const id = added.payload.zoomId; + const shown = run(added.document as AxcutDocument, "setZoom", { + zoomId: id, + hideCursor: false, + focusMode: "manual", + }); + expect(shown.document?.zoomRanges[0]).not.toHaveProperty("hideCursor"); + expect(shown.document?.zoomRanges[0]).toMatchObject({ focusMode: "manual", customScale: 1.6 }); + expect(snapshot(shown.document).zoomRanges[0]).not.toHaveProperty("hideCursor"); + }); + + it("writes a level the depth table has as that depth, as the inspector does", () => { + const added = run(fixtureDocument(), "addZoom", { startSec: 2, endSec: 4, customScale: 2.2 }); + expect(added.document?.zoomRanges[0]).toMatchObject({ depth: 4 }); + expect(added.document?.zoomRanges[0]).not.toHaveProperty("customScale"); + + const custom = run(added.document as AxcutDocument, "setZoom", { + zoomId: added.payload.zoomId, + customScale: 1.333, + }); + // To the hundredth, like a typed level. + expect(custom.document?.zoomRanges[0]).toMatchObject({ depth: 4, customScale: 1.33 }); + const preset = run(custom.document as AxcutDocument, "setZoom", { + zoomId: added.payload.zoomId, + customScale: 1.5, + }); + expect(preset.document?.zoomRanges[0]).toMatchObject({ depth: 2 }); + expect(preset.document?.zoomRanges[0]).not.toHaveProperty("customScale"); + }); + + it("refuses a level past the clip's own limit, naming it, and depth with customScale", () => { + const added = run(fixtureDocument(), "addZoom", { startSec: 2, endSec: 4 }); + const limit = zoomScaleLimit(added.document as AxcutDocument, added.payload.zoomId); + // The fixture's 1080p take at the default padding blurs before the renderer's own 5×. + expect(limit).toBeLessThan(5); + const past = run(added.document as AxcutDocument, "setZoom", { + zoomId: added.payload.zoomId, + customScale: limit + 0.1, + }); + expect(past.ok).toBe(false); + expect(past.payload.error).toContain(`${limit}×`); + expect(past.document).toBeUndefined(); + expect( + run(fixtureDocument(), "addZoom", { startSec: 2, endSec: 4, customScale: limit + 0.1 }).ok, + ).toBe(false); + expect( + run(added.document as AxcutDocument, "setZoom", { + zoomId: added.payload.zoomId, + customScale: limit, + }).ok, + ).toBe(true); + const both = run(fixtureDocument(), "addZoom", { + startSec: 2, + endSec: 4, + depth: 3, + customScale: 2, + }); + expect(both.ok).toBe(false); + }); + + it("reads an annotation back in the terms its writes take", () => { + const added = run(fixtureDocument(), "addAnnotation", { + startSec: 1, + endSec: 3, + text: "Hi", + x: 30, + y: 70, + fontSize: 48, + background: "none", + animation: "pop", + }); + const [ann] = snapshot(added.document).annotations; + expect(ann).toMatchObject({ + type: "text", + text: "Hi", + box: "frame", + x: expect.closeTo(30, 1), + y: expect.closeTo(70, 1), + fontSize: 48, + color: "#ffffff", + background: "none", + bold: true, + animation: "pop", + }); + expect(ann.width).toBeGreaterThan(0); + expect(added.payload).toMatchObject({ type: "text", fontSize: 48 }); + }); + + it("adds a privacy blur on the recording's picture, sized and placed as asked", () => { + const added = run(fixtureDocument(), "addAnnotation", { + startSec: 1, + endSec: 3, + type: "blur", + x: 20, + y: 10, + width: 30, + height: 6, + blurData: { type: "blur", intensity: 20 }, + }); + expect(added.ok).toBe(true); + const stored = added.document?.annotations[0]; + expect(stored).toMatchObject({ + type: "blur", + content: "", + position: { x: 5, y: 7 }, + size: { width: 30, height: 6 }, + blurData: { type: "blur", shape: "rectangle", intensity: 20 }, + }); + // A blur stays on what it hides: measured on the footage, never on the frame. + expect(stored?.space).toBeUndefined(); + expect(() => documentSchema.parse(added.document)).not.toThrow(); + expect(snapshot(added.document).annotations[0]).toMatchObject({ + type: "blur", + box: "footage", + x: 20, + y: 10, + width: 30, + height: 6, + blurData: { type: "blur", shape: "rectangle", color: "white", intensity: 20, blockSize: 12 }, + }); + expect(snapshot(added.document).annotations[0]).not.toHaveProperty("text"); + + // A patch of blurData changes what it names and keeps the rest. + const oval = run(added.document as AxcutDocument, "setAnnotation", { + annotationId: added.payload.annotationId, + blurData: { shape: "oval" }, + }); + expect(oval.document?.annotations[0].blurData).toMatchObject({ + type: "blur", + shape: "oval", + intensity: 20, + }); + }); + + it("adds an arrow on the frame with its figureData", () => { + const added = run(fixtureDocument(), "addAnnotation", { + startSec: 1, + endSec: 3, + type: "figure", + figureData: { arrowDirection: "down-left", color: "#ff0000" }, + }); + expect(added.ok).toBe(true); + expect(added.document?.annotations[0]).toMatchObject({ + type: "figure", + space: "frame", + size: { width: 30, height: 20 }, + figureData: { arrowDirection: "down-left", color: "#ff0000", strokeWidth: 4 }, + }); + expect(() => documentSchema.parse(added.document)).not.toThrow(); + expect(snapshot(added.document).annotations[0]).toMatchObject({ + type: "figure", + box: "frame", + x: 50, + y: 50, + figureData: { arrowDirection: "down-left" }, + }); + }); + + it("switches an annotation's type as the inspector does, keeping the text to come back to", () => { + const added = run(fixtureDocument(), "addAnnotation", { + startSec: 1, + endSec: 3, + text: "secret@example.com", + }); + const id = added.payload.annotationId; + const blurred = run(added.document as AxcutDocument, "setAnnotation", { + annotationId: id, + type: "blur", + }); + expect(blurred.ok).toBe(true); + const blur = blurred.document?.annotations[0]; + expect(blur).toMatchObject({ type: "blur", content: "", textContent: "secret@example.com" }); + expect(blur?.space).toBeUndefined(); + expect(() => documentSchema.parse(blurred.document)).not.toThrow(); + + const back = run(blurred.document as AxcutDocument, "setAnnotation", { + annotationId: id, + type: "text", + }); + expect(back.document?.annotations[0]).toMatchObject({ + type: "text", + content: "secret@example.com", + space: "frame", + }); + }); + + it("refuses arguments that mean nothing for the annotation's kind", () => { + const text = run(fixtureDocument(), "addAnnotation", { startSec: 1, endSec: 3, text: "Hi" }); + const id = text.payload.annotationId; + const doc = text.document as AxcutDocument; + expect(run(doc, "setAnnotation", { annotationId: id, width: 40 }).ok).toBe(false); + expect(run(doc, "setAnnotation", { annotationId: id, blurData: { type: "blur" } }).ok).toBe( + false, + ); + expect( + run(doc, "setAnnotation", { annotationId: id, type: "blur", blurData: { type: "blur" } }).ok, + ).toBe(true); + expect( + run(fixtureDocument(), "addAnnotation", { + startSec: 1, + endSec: 3, + type: "blur", + blurData: { shape: "freehand" }, + }).ok, + ).toBe(false); + expect( + run(fixtureDocument(), "addAnnotation", { startSec: 1, endSec: 3, type: "image" }).ok, + ).toBe(false); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index c63a56d4c..9e01e253b 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -17,8 +17,13 @@ import { z } from "zod"; import { DEFAULT_TEXT_PLATE } from "../../src/lib/ai-edition/annotations/background"; -import { fitTextInFrame } from "../../src/lib/ai-edition/annotations/placement"; import { + convertAnnotationKind, + DEFAULT_ANNOTATION_BOX, + fitTextInFrame, +} from "../../src/lib/ai-edition/annotations/placement"; +import { + AUDIO_FADE_MAX_MS, collapseTracksToPills, patchAudioTrack, placeAudioTrackInDocument, @@ -36,7 +41,12 @@ import { setClipSourceRange, } from "../../src/lib/ai-edition/document/timeline"; import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript"; -import type { AxcutAnnotationRegion, AxcutDocument } from "../../src/lib/ai-edition/schema"; +import { + type AxcutAnnotationRegion, + type AxcutDocument, + blurDataSchema, + figureDataSchema, +} from "../../src/lib/ai-edition/schema"; import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings"; import { applySettingsPatch } from "../../src/lib/ai-edition/store/settingsPatch"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; @@ -58,7 +68,10 @@ import { } from "../../src/lib/ai-edition/timeline/trim-mapping"; import { effectiveZoomScale, + MAX_ZOOM_SCALE, + MIN_ZOOM_SCALE, ZOOM_DEPTH_LEGEND, + zoomLevelFor, } from "../../src/lib/ai-edition/timeline/zoom-scale"; import { chainedPans, @@ -66,6 +79,7 @@ import { zoomTransitions, } from "../../src/lib/ai-edition/timeline/zoom-transitions"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; +import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription"; import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils"; import { measureTextInShippedFonts } from "./text-measure"; @@ -198,6 +212,129 @@ function applyAnnotationLook( return next; } +const round2 = (n: number) => Math.round(n * 100) / 100; + +/** + * One annotation as the agent reads it back, in the terms its writes take: the centre (x, y) and + * size of its box, which box that is, and the look of its kind. A text reports the look arguments, + * an arrow its figureData and a blur its blurData, the defaults filled in as the renderer fills + * them. An image reports its box only: its content is a data URL, megabytes of nothing to read. + */ +function annotationForAgent(a: AxcutAnnotationRegion): Record { + const geometry = { + box: a.space === "frame" ? "frame" : "footage", + x: round2(a.position.x + a.size.width / 2), + y: round2(a.position.y + a.size.height / 2), + width: round2(a.size.width), + height: round2(a.size.height), + }; + switch (a.type) { + case "text": + return { + type: a.type, + text: a.content || a.textContent || "", + ...geometry, + fontSize: a.style.fontSize, + color: a.style.color, + background: a.style.backgroundColor === "transparent" ? "none" : a.style.backgroundColor, + bold: a.style.fontWeight === "bold", + animation: a.style.textAnimation ?? "none", + }; + case "figure": + return { + type: a.type, + ...geometry, + figureData: figureDataSchema.unwrap().parse(a.figureData ?? {}), + }; + case "blur": + return { + type: a.type, + ...geometry, + blurData: blurDataSchema.unwrap().parse(a.blurData ?? {}), + }; + default: + return { type: a.type, ...geometry }; + } +} + +/** The arguments that only mean something for one kind of annotation, refused on another. */ +function refuseShapeMismatch( + type: AxcutAnnotationRegion["type"], + data: { + text?: string; + width?: number; + height?: number; + figureData?: unknown; + blurData?: { shape?: string }; + }, +): AgentToolExecution | null { + if (type === "text" && (data.width !== undefined || data.height !== undefined)) { + return failure("A text box is sized to its words: change fontSize, not width or height."); + } + if (type !== "text" && data.text !== undefined) { + return failure(`text is for a text annotation; this one is ${type}. Pass type "text" with it.`); + } + if (type !== "figure" && data.figureData !== undefined) { + return failure(`figureData is an arrow's; this annotation is ${type}. Pass type "figure".`); + } + if (type !== "blur" && data.blurData !== undefined) { + return failure(`blurData is a blur's; this annotation is ${type}. Pass type "blur".`); + } + // The editor no longer offers it either: its input was broken and it renders as its bounding + // box, and a privacy mask that is half reliable is worse than none. + if (data.blurData?.shape === "freehand") { + return failure('A freehand blur is not offered: use shape "rectangle" or "oval".'); + } + return null; +} + +/** The fields of `parsed` the caller actually sent: the schema fills in defaults, and a patch + * must not reset what it never named. */ +function sentKeys(parsed: T | undefined, raw: unknown): Partial { + if (!parsed || !raw || typeof raw !== "object") return {}; + return Object.fromEntries(Object.entries(parsed).filter(([key]) => key in raw)) as Partial; +} + +/** + * An arrow's, a blur's or an image's box and data: `size` centred on `center` and kept inside its + * reference box, as the editor keeps one it moves; then the figureData or blurData the caller sent, + * over what the annotation had or the schema's defaults. + */ +function shapeAnnotation( + region: AxcutAnnotationRegion, + center: { x: number; y: number }, + size: { width: number; height: number }, + data: { figureData?: object; blurData?: object }, + rawArgs: unknown, +): AxcutAnnotationRegion { + const raw = (rawArgs ?? {}) as { figureData?: unknown; blurData?: unknown }; + const { width, height } = size; + return { + ...region, + position: { + x: Math.min(Math.max(center.x - width / 2, 0), Math.max(0, 100 - width)), + y: Math.min(Math.max(center.y - height / 2, 0), Math.max(0, 100 - height)), + }, + size: { width, height }, + ...(region.type === "figure" + ? { + figureData: { + ...figureDataSchema.unwrap().parse(region.figureData ?? {}), + ...sentKeys(data.figureData, raw.figureData), + }, + } + : {}), + ...(region.type === "blur" + ? { + blurData: { + ...blurDataSchema.unwrap().parse(region.blurData ?? {}), + ...sentKeys(data.blurData, raw.blurData), + }, + } + : {}), + } as AxcutAnnotationRegion; +} + /** * A text annotation on the output frame, its box sized to its text around `center` (frame * percentages): what the editor's "add text" does, so the agent's text sits where the editor's @@ -584,12 +721,28 @@ const focusSchema = z.object({ cx: z.number().min(0).max(1), cy: z.number().min( * (getCurrentDocument) as `rotationPreset`, absent when flat. */ const rotationPresetSchema = z.enum(["none", "left", "right", "orbit"]); +/** The inspector's focus mode: `manual` holds `focus`, `auto` follows the recorded pointer. */ +const focusModeSchema = z.enum(["manual", "auto"]); + +/** The inspector's free zoom level: a factor, refused outside the range it accepts. The clip's own + * limit (`zoomScaleLimit`) is checked once the zoom is placed, as the inspector checks it. */ +const customScaleSchema = z + .number() + .min(MIN_ZOOM_SCALE) + .max(MAX_ZOOM_SCALE) + .describe( + `A free zoom level, as a factor (${MIN_ZOOM_SCALE}–${MAX_ZOOM_SCALE}, e.g. 2 for 2×), in place of depth. Refused past the clip's own limit, the deepest its recording takes before it blurs; the refusal names it.`, + ); + export const addZoomArgs = z.object({ startSec: secondsSchema, endSec: secondsSchema, depth: depthSchema.default(3), focus: focusSchema.default({ cx: 0.5, cy: 0.5 }), rotationPreset: rotationPresetSchema.default("none"), + focusMode: focusModeSchema.default("manual"), + hideCursor: z.boolean().default(false), + customScale: customScaleSchema.optional(), }); /** A zoom with its 3D camera set: `none` drops the key, as a flat zoom never carries it. */ @@ -601,6 +754,59 @@ function withRotationPreset( return (preset === "none" ? flat : { ...flat, rotationPreset: preset }) as Z; } +/** `customScale` as the inspector's typed level writes it: to the hundredth, and a level the depth + * table has as that depth (`zoomLevelFor`). Empty when the caller passed none. */ +function zoomLevelArg(customScale: number | undefined): { + depth?: 1 | 2 | 3 | 4 | 5 | 6; + customScale?: number; +} { + return customScale === undefined ? {} : zoomLevelFor(Math.round(customScale * 100) / 100); +} + +/** depth and customScale are two ways of saying one thing: both at once has no single meaning. */ +function refuseTwoLevels(args: unknown): AgentToolExecution | null { + const raw = (args ?? {}) as { depth?: unknown; customScale?: unknown }; + return raw.depth !== undefined && raw.customScale !== undefined + ? failure("Pass depth or customScale, not both: they set the same zoom level.") + : null; +} + +/** + * The inspector's bound on a typed level: the deepest the clip under the zoom takes before its + * recording blurs (`zoomScaleLimit`, the bound the scene applies), the tightest over every + * fragment. Refused rather than clamped, as the inspector refuses it, so the caller hears the limit. + */ +function refuseScalePastLimit( + next: AxcutDocument, + ids: string[], + customScale: number | undefined, +): AgentToolExecution | null { + if (customScale === undefined) return null; + const scale = Math.round(customScale * 100) / 100; + const limit = Math.min(...ids.map((id) => zoomScaleLimit(next, id))); + return scale > limit + ? failure( + `The zoom was NOT written: customScale ${scale} is deeper than this clip takes (${limit}× at most), past which its recording blurs. Pick ${limit} or less.`, + ) + : null; +} + +/** focusMode, hideCursor and customScale as the snapshot reports them, for a write's result. */ +function zoomBehaviourReport( + zoom: AxcutDocument["zoomRanges"][number], + document: AxcutDocument, +): Record { + const autoFocusAll = (document.legacyEditor as Record | null)?.autoFocusAll; + return { + focusMode: autoFocusAll === true ? "auto" : (zoom.focusMode ?? "manual"), + ...(autoFocusAll === true && zoom.focusMode !== "auto" + ? { focusModeOverriddenByAutoFocusAll: true } + : {}), + ...(zoom.hideCursor ? { hideCursor: true } : {}), + ...(zoom.customScale != null ? { customScale: zoom.customScale } : {}), + }; +} + /** Same contract as `addTrimsArgs`: the element schema IS the unitary one, and * it is advertised rather than enforced so a bad region is refused by itself. */ export const addZoomsArgs = z.object({ @@ -614,6 +820,9 @@ export const setZoomArgs = z.object({ depth: depthSchema.optional(), focus: focusSchema.optional(), rotationPreset: rotationPresetSchema.optional(), + focusMode: focusModeSchema.optional(), + hideCursor: z.boolean().optional(), + customScale: customScaleSchema.optional(), }); // The bound every speed reader applies (`readSpeedRegions`): refused here, so the agent hears @@ -656,25 +865,61 @@ const annotationLookArgs = { .describe("How the text enters: typewriter types it out, pop springs it in, and so on."), }; +/** The kinds the inspector's type switch offers, image aside: an image needs a file read. */ +const annotationTypeSchema = z + .enum(["text", "figure", "blur"]) + .describe( + "text: words on the frame. figure: an arrow. blur: a privacy mask that blurs or pixelates what is under it, such as an email address or a token on screen.", + ); + +/** What an arrow or a blur has instead of a text's look: its box, and its own data. */ +const annotationShapeArgs = { + width: z + .number() + .positive() + .max(100) + .optional() + .describe( + "Box width, in percent of the annotation's box: the output frame for an arrow, the recording's picture for a blur. Not for text, whose box is its words.", + ), + height: z + .number() + .positive() + .max(100) + .optional() + .describe("Box height, in percent of the same box as width."), + figureData: figureDataSchema.describe("An arrow's direction, colour and stroke width."), + blurData: blurDataSchema.describe( + "A blur's look: type blur or mosaic, shape rectangle or oval, color of the mosaic grid, intensity, blockSize.", + ), +}; + export const addAnnotationArgs = z.object({ startSec: secondsSchema, endSec: secondsSchema, + type: annotationTypeSchema.default("text"), text: z.string().default(""), x: z.number().min(0).max(100).default(50), y: z.number().min(0).max(100).default(50), ...annotationLookArgs, + ...annotationShapeArgs, }); export const setAnnotationArgs = z.object({ annotationId: z.string().min(1), startSec: secondsSchema.optional(), endSec: secondsSchema.optional(), + type: annotationTypeSchema.optional(), text: z.string().optional(), x: z.number().min(0).max(100).optional(), y: z.number().min(0).max(100).optional(), ...annotationLookArgs, + ...annotationShapeArgs, }); +/** A track's fade, capped where the inspector's slider stops. */ +const fadeMsSchema = z.number().int().min(0).max(AUDIO_FADE_MAX_MS); + export const addAudioArgs = z.object({ assetId: z.string().min(1), startSec: secondsSchema, @@ -682,6 +927,8 @@ export const addAudioArgs = z.object({ kind: z.enum(["voiceover", "music"]).default("music"), offsetSec: secondsSchema.default(0), gainDb: z.number().min(-60).max(12).default(0), + fadeInMs: fadeMsSchema.default(0), + fadeOutMs: fadeMsSchema.default(0), }); export const setAudioArgs = z.object({ @@ -693,6 +940,8 @@ export const setAudioArgs = z.object({ gainDb: z.number().min(-60).max(12).optional(), muted: z.boolean().optional(), loop: z.boolean().optional(), + fadeInMs: fadeMsSchema.optional(), + fadeOutMs: fadeMsSchema.optional(), }); export const addCameraFullscreenArgs = z.object({ @@ -996,7 +1245,10 @@ export function documentSnapshotForModel( "timeline, longer above 1x and shorter below, so read the bounds rather than adding transitionSec). cutByTrim names the move a trim cuts into (in, out or both), cutByJunction the move a clip junction cuts (the clip ends before the move does): the export jumps at that cut. " + "Two zooms at most 1.5 s apart on screen, with no trim or clip junction between them, are CHAINED: " + "no zoom-out and back in, the camera pans from the first (panToZoomId, panUntilSec: 1 s of screen " + - "time from its endSec, which can run into the next span) to the second (panFromZoomId).", + "time from its endSec, which can run into the next span) to the second (panFromZoomId). " + + "focusMode auto follows the recorded pointer instead of holding focus; hideCursor: true hides the pointer while the zoom is on screen (absent, it shows).", + annotationNote: + "Each annotation reports its box the way addAnnotation and setAnnotation take it: x/y its centre, width/height its size, in percent of `box`. box 'frame' is the output frame (text and arrows, which padding and zooms never move); box 'footage' is the recording's picture (a blur, which stays on what it hides when the picture moves). A text also reports its look (fontSize, color, background, bold, animation), an arrow (type 'figure') its figureData, a blur its blurData.", project: { id: document.project.id, title: document.project.title }, primaryAssetId: document.project.primaryAssetId ?? document.assets[0]?.id ?? null, autoFocusAll, @@ -1074,6 +1326,7 @@ export function documentSnapshotForModel( ...(z.rotationPreset ? { rotationPreset: z.rotationPreset } : {}), focus: z.focus, focusMode: autoFocusAll ? "auto" : (z.focusMode ?? "manual"), + ...(z.hideCursor ? { hideCursor: true } : {}), source: z.source ?? "manual", }), ), @@ -1087,8 +1340,7 @@ export function documentSnapshotForModel( id: a.id, startSec: roundSec(a.startMs), endSec: roundSec(a.endMs), - type: a.type, - text: a.textContent ?? a.content ?? "", + ...annotationForAgent(a), })), cameraFullscreenRegions: coalesceForAgent(cameraFullscreenRegions).map((c) => ({ id: c.id, @@ -1110,6 +1362,8 @@ export function documentSnapshotForModel( gainDb: t.gainDb, muted: t.muted, loop: t.loop, + fadeInMs: t.fadeInMs, + fadeOutMs: t.fadeOutMs, })), hasTranscript: document.transcripts.length > 0 || document.transcript !== null, }; @@ -2170,16 +2424,21 @@ export function executeAgentTool( case "addZoom": { const parsed = addZoomArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); + const twoLevels = refuseTwoLevels(args); + if (twoLevels) return twoLevels; const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); + const level = zoomLevelArg(parsed.data.customScale); const zoom = withRotationPreset( { id: createId("zoom"), startMs, endMs, depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6, + ...level, focus: parsed.data.focus, - focusMode: "manual" as const, + focusMode: parsed.data.focusMode, + ...(parsed.data.hideCursor ? { hideCursor: true } : {}), source: "manual" as const, } as AxcutDocument["zoomRanges"][number], parsed.data.rotationPreset, @@ -2191,6 +2450,8 @@ export function executeAgentTool( ...document, zoomRanges: [...document.zoomRanges, ...placed] as AxcutDocument["zoomRanges"], }; + const pastLimit = refuseScalePastLimit(next, landing.ids, parsed.data.customScale); + if (pastLimit) return pastLimit; // Measured over what was STORED, never over what was asked for: `placed` // is the clamped, ventilated truth, so the report cannot end up // describing a window the zoom does not occupy. @@ -2205,6 +2466,7 @@ export function executeAgentTool( // The depth alone is an ordinal; reported on its own it is what the // model turns into "3×" for a frame that renders 1.80×. renderedScale: effectiveZoomScale(zoom), + ...zoomBehaviourReport(zoom, document), ...landingReport(landing, startMs / 1000, endMs / 1000), ...(anchor ? { cursorAnchor: anchor } : {}), ...cutTransitionsReport(document, next), @@ -2225,6 +2487,8 @@ export function executeAgentTool( case "setZoom": { const parsed = setZoomArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); + const twoLevels = refuseTwoLevels(args); + if (twoLevels) return twoLevels; const { zoomId } = parsed.data; const existing = document.zoomRanges.find((z) => z.id === zoomId); if (!existing) return failure(`Unknown zoom: ${zoomId}`); @@ -2244,17 +2508,26 @@ export function executeAgentTool( const clearsCustomScale = parsed.data.depth !== undefined && document.zoomRanges.some((z) => zoomPill.has(z.id) && z.customScale != null); + const level = zoomLevelArg(parsed.data.customScale); + const { hideCursor, focusMode } = parsed.data; const rebuiltZooms = replacePillSpan( // payload edits first, applied to every region under the pill… document.zoomRanges.map((z) => { if (!zoomPill.has(z.id)) return z; - const { customScale, ...rest } = z; + const { customScale, hideCursor: wasHidden, ...rest } = z; + // A level the table has is written as its depth, which drops the custom scale. + const keepsScale = !clearsCustomScale && !("depth" in level); const edited = { - ...(clearsCustomScale ? rest : z), + ...rest, + ...(keepsScale && customScale != null ? { customScale } : {}), + // As the inspector writes it: shown is the absence of the flag. + ...((hideCursor ?? wasHidden) ? { hideCursor: true } : {}), ...(parsed.data.depth !== undefined ? { depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6 } : {}), + ...level, ...(parsed.data.focus ? { focus: parsed.data.focus } : {}), + ...(focusMode ? { focusMode } : {}), }; return parsed.data.rotationPreset === undefined ? edited @@ -2270,6 +2543,8 @@ export function executeAgentTool( const landing = landingAfterPillEdit(document.zoomRanges, rebuiltZooms, zoomPill, document); if (!landing.anchored) return coversNoClip("zoom", startMs / 1000, endMs / 1000, document); const next: AxcutDocument = { ...document, zoomRanges: rebuiltZooms }; + const pastLimit = refuseScalePastLimit(next, landing.ids, parsed.data.customScale); + if (pastLimit) return pastLimit; // Read back off the document, not off the request: the pill may have been // re-ventilated, and `renderedScale` is the only number the viewer sees. const landed = new Set(landing.ids); @@ -2296,6 +2571,7 @@ export function executeAgentTool( depth: strength.depth, renderedScale: effectiveZoomScale(strength), ...(strength.rotationPreset ? { rotationPreset: strength.rotationPreset } : {}), + ...zoomBehaviourReport(strength, document), } : {}), ...(clearsCustomScale ? { clearedCustomScale: true } : {}), @@ -2393,11 +2669,17 @@ export function executeAgentTool( case "addAnnotation": { const parsed = addAnnotationArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); + const { type } = parsed.data; + const mismatch = refuseShapeMismatch(type, { + ...parsed.data, + text: parsed.data.text || undefined, + }); + if (mismatch) return mismatch; const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); // The editor's new text: its defaults, then whatever look the caller asked for, on the // output frame with the box fitted to the text around x/y (the text's centre). - const ann = placeTextOnFrame( + const text = placeTextOnFrame( { id: createId("ann"), startMs, @@ -2426,6 +2708,29 @@ export function executeAgentTool( { x: parsed.data.x, y: parsed.data.y }, document, ); + // An arrow or a blur is that text switched to its kind, as the inspector's type switch + // does it, then given its box around x/y and its data. Nothing to convert through: the + // box is set right after, in the kind's own reference box. + const ann = + type === "text" + ? text + : shapeAnnotation( + { + ...text, + ...convertAnnotationKind(text, type, { + footage: null, + frameAspect: 0, + measure: measureTextInShippedFonts, + }), + }, + { x: parsed.data.x, y: parsed.data.y }, + { + width: parsed.data.width ?? DEFAULT_ANNOTATION_BOX.width, + height: parsed.data.height ?? DEFAULT_ANNOTATION_BOX.height, + }, + parsed.data, + args, + ); const placed = anchorForAgent(ann, document, "ann"); const landing = landingOf(placed, document); if (!landing.anchored) { @@ -2441,9 +2746,13 @@ export function executeAgentTool( resultJson: JSON.stringify({ annotationId: landing.ids[0], ...landingReport(landing, startMs / 1000, endMs / 1000), + ...annotationForAgent(placed[0] as AxcutAnnotationRegion), }), summary: - `added annotation "${parsed.data.text.slice(0, 24)}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + (type === "text" + ? `added annotation "${parsed.data.text.slice(0, 24)}"` + : `added ${type === "blur" ? "blur" : "arrow"}`) + + ` ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + landingSuffix(landing, startMs / 1000, endMs / 1000), }; } @@ -2461,8 +2770,28 @@ export function executeAgentTool( endMs, }); if (blocked) return blocked; - const { x, y, text } = parsed.data; - const edit = (a: AxcutAnnotationRegion): AxcutAnnotationRegion => { + const { x, y, text, type, width, height } = parsed.data; + const mismatch = refuseShapeMismatch(type ?? existing.type, parsed.data); + if (mismatch) return mismatch; + const frameAspect = resolveAspectRatioValue( + document, + getEditorSettings(document).aspectRatio, + ); + const edit = (original: AxcutAnnotationRegion): AxcutAnnotationRegion => { + // The inspector's type switch, through the footage under this very fragment: the box + // changes reference (a blur lives on the footage, the rest on the frame) without + // moving on screen, and the previous kind's content is parked, not lost. + const converted = type !== undefined && type !== original.type; + const a = converted + ? { + ...original, + ...convertAnnotationKind(original, type, { + footage: annotationFootageRect(document, original.id), + frameAspect, + measure: measureTextInShippedFonts, + }), + } + : original; const next: AxcutAnnotationRegion = { ...a, ...(text !== undefined ? { content: text, textContent: text } : {}), @@ -2485,16 +2814,23 @@ export function executeAgentTool( ? placeTextOnFrame(next, centre, document) : next; } - if (!moved) return next; - // An image or an arrow keeps its size and moves its centre, inside its own box. - const { width, height } = a.size; - return { - ...next, - position: { - x: Math.min(Math.max(centre.x - width / 2, 0), Math.max(0, 100 - width)), - y: Math.min(Math.max(centre.y - height / 2, 0), Math.max(0, 100 - height)), - }, - }; + const reshaped = + moved || + converted || + width !== undefined || + height !== undefined || + parsed.data.figureData !== undefined || + parsed.data.blurData !== undefined; + if (!reshaped) return next; + // An image, an arrow or a blur keeps its size unless given one, and moves its centre, + // inside its own box. + return shapeAnnotation( + next, + centre, + { width: width ?? a.size.width, height: height ?? a.size.height }, + parsed.data, + args, + ); }; const rebuiltAnnotations = replacePillSpan( document.annotations.map((a) => (annPill.has(a.id) ? edit(a) : a)), @@ -2514,12 +2850,14 @@ export function executeAgentTool( return coversNoClip("annotation", startMs / 1000, endMs / 1000, document); } const next: AxcutDocument = { ...document, annotations: rebuiltAnnotations }; + const landed = rebuiltAnnotations.find((a) => a.id === landing.ids[0]); return { ok: true, document: next, resultJson: JSON.stringify({ annotationId: landing.ids[0] ?? annotationId, ...landingReport(landing, startMs / 1000, endMs / 1000), + ...(landed ? annotationForAgent(landed) : {}), }), summary: `updated annotation ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + @@ -2611,7 +2949,7 @@ export function executeAgentTool( case "addAudio": { const parsed = addAudioArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); - const { assetId, kind, offsetSec, gainDb } = parsed.data; + const { assetId, kind, offsetSec, gainDb, fadeInMs, fadeOutMs } = parsed.data; const asset = document.assets.find((a) => a.id === assetId); // Two distinct refusals, because they need two different corrections: an // unknown id is a hallucinated asset, a video id is the model reaching for @@ -2662,8 +3000,8 @@ export function executeAgentTool( offsetMs: toMs(offsetSec), gainDb, loop: false, - fadeInMs: 0, - fadeOutMs: 0, + fadeInMs, + fadeOutMs, muted: false, label: asset.label, origin: "agent", @@ -2716,6 +3054,8 @@ export function executeAgentTool( ...(parsed.data.gainDb !== undefined ? { gainDb: parsed.data.gainDb } : {}), ...(parsed.data.muted !== undefined ? { muted: parsed.data.muted } : {}), ...(parsed.data.loop !== undefined ? { loop: parsed.data.loop } : {}), + ...(parsed.data.fadeInMs !== undefined ? { fadeInMs: parsed.data.fadeInMs } : {}), + ...(parsed.data.fadeOutMs !== undefined ? { fadeOutMs: parsed.data.fadeOutMs } : {}), ...(parsed.data.offsetSec !== undefined ? { offsetMs: toMs(parsed.data.offsetSec) } : {}), }); @@ -2762,6 +3102,8 @@ export function executeAgentTool( audioId, startSec: roundSec(after?.startMs ?? pill.startMs), endSec: roundSec(after?.endMs ?? pill.endMs), + fadeInMs: after?.fadeInMs ?? pill.fadeInMs, + fadeOutMs: after?.fadeOutMs ?? pill.fadeOutMs, }), summary: `updated audio ${audioId} ${formatSec(roundSec(after?.startMs ?? pill.startMs))} – ${formatSec(roundSec(after?.endMs ?? pill.endMs))}`, }; diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 28390d4bf..d14b3e9ef 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -18,6 +18,7 @@ import { ChatAnthropic } from "@langchain/anthropic"; import type { StructuredToolInterface } from "@langchain/core/tools"; import { ChatOpenAI } from "@langchain/openai"; import { describe, expect, it } from "vitest"; +import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks"; import { type AxcutDocument, createEmptyDocument, @@ -388,6 +389,17 @@ describe("what the descriptions say about zoom strength", () => { expect(ZOOM_DEPTH_LEGEND).toContain(`3=${ZOOM_DEPTH_SCALES[3].toFixed(2)}×`); }); + it("offers the blur as the way to hide a private detail, and says images are not here (#1180)", () => { + expect(TOOL_DESCRIPTIONS.addAnnotation).toMatch(/'blur'.*email/); + expect(TOOL_DESCRIPTIONS.addAnnotation).toMatch(/Image annotations are not available/); + expect(SYSTEM_PROMPT).toMatch(/privacy blur/); + for (const name of ["addZoom", "setZoom"]) { + expect(TOOL_DESCRIPTIONS[name]).toMatch(/hideCursor/); + expect(TOOL_DESCRIPTIONS[name]).toMatch(/customScale/); + } + expect(TOOL_DESCRIPTIONS.setAudio).toContain(`${AUDIO_FADE_MAX_MS} ms`); + }); + it("tells the model a full-camera region needs a linked webcam", () => { expect(TOOL_DESCRIPTIONS.addCameraFullscreen).toMatch(/hasCameraTrack/); expect(SYSTEM_PROMPT).toMatch(/hasCameraTrack/); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 345721a60..7b05c0444 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -19,12 +19,17 @@ import { anthropicPromptCachingMiddleware, createAgent, tool } from "langchain"; import { z } from "zod"; +import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks"; import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; // ponytail: the legend is DERIVED from the depth→scale table, not retyped. Both // tool descriptions used to assert "depth 1–6 maps to 1.0×–3.5×" — a formula // (`depth/2 + 0.5`) that was purged from two rendering sites and survived here, // wrong at both ends of a table that actually runs 1.25× to 5.0×. -import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale"; +import { + MAX_ZOOM_SCALE, + MIN_ZOOM_SCALE, + ZOOM_DEPTH_LEGEND, +} from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions"; import { AGENT_EDITS_SETTING, @@ -126,7 +131,7 @@ const BASE_SYSTEM_PROMPT = [ "- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.", "- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", - `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text on screen. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, + `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text, an arrow or a privacy blur on screen: a blur is how an email address, a token or any private detail the recording shows is hidden. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, "- setAspectRatio picks the shape of the whole video, for where it will be shown; auto follows the recording.", "- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.", "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", @@ -169,12 +174,14 @@ export const SYSTEM_PROMPT = buildSystemPrompt({ editsAllowed: true }); const ZOOM_CAMERA_NOTE = "rotationPreset is the zoom's 3D camera: `left` or `right` turns the screen in 3D to that side, `orbit` moves the camera around the screen while the zoom holds, `none` (the default) keeps it flat. It is a strong effect: keep it for a reveal or a key moment, not every zoom. getCurrentDocument reports it as rotationPreset, absent when flat."; +const ZOOM_BEHAVIOUR_NOTE = `The zoom pane's other settings: focusMode "manual" holds focus, "auto" follows the recorded pointer instead (getCurrentDocument reports the effective mode, auto everywhere while autoFocusAll is on); hideCursor hides the pointer while the zoom is on screen; customScale is a free level in place of depth, a factor from ${MIN_ZOOM_SCALE} to ${MAX_ZOOM_SCALE} rounded to the hundredth. A level the depth table has is stored as that depth, and one deeper than the clip takes before its recording blurs is refused with the limit, as the editor refuses it. Pass depth or customScale, not both.`; + const ZOOM_TRANSITIONS_NOTE = `A zoom HOLDS over its span: it animates in during the window just before startSec and out during the window just after endSec. Each move lasts 0.6 s + 0.55 s × ln(renderedScale) of SCREEN time (${ZOOM_TRANSITION_LEGEND}), so a speed region scales the timeline it covers by its speed: longer above 1×, shorter below. getCurrentDocument gives each zoom's zoomInFromSec and zoomOutUntilSec, in virtual seconds like the zoom. A trim inside either window cuts that move and the export jumps at the cut: leave the windows untrimmed. Two zooms at most 1.5 s apart on screen, on the same clip with no trim between them, are chained: the camera pans from one to the next over 1 s of screen time from the first one's end (panUntilSec), with no zoom-out or zoom-in between them.`; const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it carries \`cutTransitions\`: which zoom, which side, cutSec, the timeline seconds of the move lost, and by: trim, or junction when the move runs past its clip's edge into the next clip. It is a warning, nothing is refused: move the cut or the zoom unless a hard cut is what you want.`; export const TOOL_DESCRIPTIONS: Record = { getCurrentDocument: - "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation effects (virtual, edited-timeline time). Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", + "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", getTranscript: "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: @@ -195,27 +202,25 @@ export const TOOL_DESCRIPTIONS: Record = { "Reorder a placed clip: move `clipId` so it plays just before `beforeClipId` (pass null, or omit it, to move it last). Ids come from getCurrentDocument, where each clip carries its `index` and its label in `reason`. This preserves every clip id, every source range, every trim, and the zooms / speed regions / annotations anchored to each clip. This is the tool for 'swap these clips', 'put X first' and 'change the clip order' — replaceTimeline cannot reorder anything.", replaceTimeline: "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.", - addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, + addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_BEHAVIOUR_NOTE} ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry, and the same \`cutTransitions\` warning. ${ZOOM_TRANSITIONS_NOTE}`, - setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, + setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_BEHAVIOUR_NOTE} ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addSpeed: "Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down, from 0.25× to 16× (default 1.5×). Use to speed through slow stretches without cutting them.", setSpeed: "Move, resize, or change the multiplier of an existing speed region by id (virtual-timeline seconds). Only the fields you pass are changed.", addAnnotation: - "Add a text annotation over a span of the edited timeline (virtual seconds). It sits on the output frame, the whole picture padding included, as text added in the editor does: zooms and padding never move it. x/y place the text's centre in frame percentages (0–100, default the centre); the box is sized to the text and kept inside the frame. By default it is drawn on a dark plate so it reads on any footage; fontSize, color, background, bold and animation set its look. Use for callouts, labels and titles.", + "Add an annotation over a span of the edited timeline (virtual seconds). type picks what it is: 'text' (the default) for callouts, labels and titles; 'figure' for an arrow pointing at something; 'blur' for a privacy mask that blurs or pixelates what is under it, the way to hide an email address, a token, a password field or any private detail the recording shows. Image annotations are not available here: they need a file, added in the editor. x/y place the centre, 0–100 percent of the annotation's box (default the centre). Text and arrows sit on the output frame, the whole picture padding included, as the editor places them: zooms and padding never move them. A blur sits on the recording's picture instead, so x/y/width/height are percent of the recording and it stays on what it hides when padding or a zoom moves the picture. A text box is sized to its words and kept inside the frame; it is drawn on a dark plate by default, and fontSize, color, background, bold and animation set its look. An arrow or a blur takes width/height (default 30 × 20) and its own data: figureData {arrowDirection, color, strokeWidth} for an arrow, blurData {type 'blur' or 'mosaic', shape 'rectangle' or 'oval', color, intensity, blockSize} for a blur. The result reads the annotation back as getCurrentDocument reports it.", setAnnotation: - "Change an existing annotation by id: its span (virtual-timeline seconds), its text, its centre (x/y, frame percentages) or its look (fontSize, color, background, bold, animation). Only the fields you pass are changed; a text box is refitted to its text around its centre. A new span that would run into another annotation is refused, naming it.", + "Change an existing annotation by id: its span (virtual-timeline seconds), its type, its centre (x/y, percent of its box), its size (width/height, arrow and blur only), its text and look (text only: fontSize, color, background, bold, animation), or its figureData / blurData, of which only the keys you pass change. Only the fields you pass are changed. A type change converts it as the editor's type switch does: it stays where it is on screen, a text turned into an arrow or a blur takes the default box, and the text is kept to come back to. A text box is refitted to its text around its centre. Arguments that belong to another kind (width on a text, blurData on an arrow) are refused. A new span that would run into another annotation is refused, naming it.", setAspectRatio: "Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording. Annotations on the frame keep their place and shape.", addCameraFullscreen: "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", setCameraFullscreen: "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", - addAudio: - "Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.", - setAudio: - "Move, resize, re-level, re-lane, mute, loop or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.", + addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`, + setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`, removeTrim: "Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.", removeModifier: diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 17f335048..c0551227d 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -57,7 +57,11 @@ import { } from "@/components/video-editor/types"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { resolveCaptionLane } from "@/lib/ai-edition/captions/settings"; -import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks"; +import { + AUDIO_FADE_MAX_MS, + collapseTracksToPills, + trackGroupId, +} from "@/lib/ai-edition/document/audioTracks"; import { collectNativeFormats, formatFillAvailability, @@ -3482,11 +3486,6 @@ type TimelineApi = ReturnType; // with the file name, then the volume (a local live value during the drag, // committed as one undo step on release), then a delete button styled like the // region panes' (position and mute are edited on the lane itself). -// Longest fade the inspector offers. Past a few seconds a fade stops reading as -// a fade and starts reading as a level change, and the track's own span caps it -// anyway (`resolveFadeSecs` reduces one that does not fit). -const FADE_MAX_MS = 5000; - /** * Per-track controls for the selected imported audio track (issue #350). Shown by * the inspector in place of the facet when an audio track is selected (see @@ -3582,7 +3581,7 @@ export function AudioTrackPane({ tl, onClose }: { tl: TimelineApi; onClose?: () label={ts("audioTrack.fadeIn")} value={liveFadeIn ?? track.fadeInMs} min={0} - max={FADE_MAX_MS} + max={AUDIO_FADE_MAX_MS} step={50} decimals={0} suffix=" ms" @@ -3602,7 +3601,7 @@ export function AudioTrackPane({ tl, onClose }: { tl: TimelineApi; onClose?: () label={ts("audioTrack.fadeOut")} value={liveFadeOut ?? track.fadeOutMs} min={0} - max={FADE_MAX_MS} + max={AUDIO_FADE_MAX_MS} step={50} decimals={0} suffix=" ms" diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index d8eea283a..96070498b 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -61,6 +61,7 @@ import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { formatSeconds } from "@/lib/ai-edition/timeline/format"; import { coalescedTrimGroups } from "@/lib/ai-edition/timeline/trim-mapping"; +import { zoomLevelFor } from "@/lib/ai-edition/timeline/zoom-scale"; import { clampToBound } from "@/lib/projectDefaults"; import { annotationFootageRect, zoomScaleLimit } from "@/native/sceneDescription"; import { ColorField } from "../ColorField"; @@ -414,8 +415,6 @@ const BLUR_DEFAULTS = { blockSize: 12, } as const; -const ZOOM_DEPTHS: readonly ZoomDepth[] = [1, 2, 3, 4, 5, 6]; - // The row: the default and one step either side, plus a strong close-up. The two ends of the // table (1.25×, 5×) and every level between are one entry in the free field below, which is // why the row stays short. Labels read the table, not a formula: a formula once announced @@ -488,11 +487,11 @@ export function ZoomLevelControl({ setRequested(scale); const gen = ++genRef.current; pendingRef.current.add(gen); - const depth = ZOOM_DEPTHS.find((d) => ZOOM_DEPTH_SCALES[d] === scale); + const level = zoomLevelFor(scale); const write = - depth === undefined - ? tl.updateZoomCustomScale(region.id, scale) - : tl.updateZoomDepth(region.id, depth); + "depth" in level + ? tl.updateZoomDepth(region.id, level.depth) + : tl.updateZoomCustomScale(region.id, level.customScale); // A refused write hands the level back to the document, so the same one can be retried. const settle = (ok: boolean) => { pendingRef.current.delete(gen); diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.ts index 027f330da..83dd74a1e 100644 --- a/src/lib/ai-edition/document/audioTracks.ts +++ b/src/lib/ai-edition/document/audioTracks.ts @@ -329,6 +329,13 @@ export function patchAudioTrack( }; } +/** + * Longest fade a track is given, by the inspector and the agent alike. Past a few seconds a fade + * stops reading as a fade and starts reading as a level change, and the track's own span caps it + * anyway (`resolveFadeSecs` reduces one that does not fit). + */ +export const AUDIO_FADE_MAX_MS = 5000; + /** * Fade lengths in seconds, reduced to fit inside `spanSec`. * diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index 4d8caf600..05edecab0 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -390,7 +390,7 @@ export const timelineOperationSchema = z.discriminatedUnion("type", [ // (AnnotationRegion / ZoomRegion) — duplicated here so the schema package has no // dependency on the React editor module. -const blurDataSchema = z +export const blurDataSchema = z .object({ type: z.enum(["blur", "mosaic"]).default("mosaic"), shape: z.enum(["rectangle", "oval", "freehand"]).default("rectangle"), @@ -403,7 +403,7 @@ const blurDataSchema = z }) .optional(); -const figureDataSchema = z +export const figureDataSchema = z .object({ arrowDirection: z .enum(["up", "down", "left", "right", "up-right", "up-left", "down-right", "down-left"]) diff --git a/src/lib/ai-edition/timeline/zoom-scale.ts b/src/lib/ai-edition/timeline/zoom-scale.ts index 2cc8b23bf..b1baea484 100644 --- a/src/lib/ai-edition/timeline/zoom-scale.ts +++ b/src/lib/ai-edition/timeline/zoom-scale.ts @@ -59,6 +59,18 @@ export function effectiveZoomScale(region: ZoomScaleInput): number { return ZOOM_DEPTH_SCALES[region.depth]; } +/** + * A zoom level as the document stores it: a level the table has is written as its depth, so a + * document keeps naming its presets, and anything else as a `customScale`. The inspector's level + * control and the agent's zoom tools both write through this. + */ +export function zoomLevelFor(scale: number): { depth: ZoomDepth } | { customScale: number } { + const depth = (Object.keys(ZOOM_DEPTH_SCALES).map(Number) as ZoomDepth[]).find( + (d) => ZOOM_DEPTH_SCALES[d] === scale, + ); + return depth === undefined ? { customScale: scale } : { depth }; +} + /** * How far a recording may be blown up on the finished frame before it visibly blurs: output * pixels per source pixel. Crop and zoom multiply, so the zoom a clip can take depends on From 9ffcb4e9aa7edd10b07b372f69a12ff4f9fcf5d2 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:07:12 +0200 Subject: [PATCH 2/2] docs: region tools reach arrows, blurs, zoom settings and audio fades Part of #1180 --- skills/openscreen-demo/SKILL.md | 3 ++- technical-documentation/architecture/ai-agent.md | 10 +++++----- website/docs/ai-editing.md | 2 +- .../current/ai-editing.md | 2 +- .../current/ai-editing.md | 2 +- .../current/ai-editing.md | 2 +- .../current/ai-editing.md | 2 +- .../current/ai-editing.md | 2 +- .../current/ai-editing.md | 2 +- .../current/ai-editing.md | 2 +- 10 files changed, 15 insertions(+), 14 deletions(-) diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 13272926b..5baf48627 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -81,7 +81,8 @@ Use the document tools, always with the `projectId`. Read the project first with - **Format first:** `setAspectRatio` for where the video will be shown, `16:9` for X. Do it before placing text. - **Cut dead time:** the setup at the start, the stop at the end, any wait. Use `addTrims`, or `addSpeed` to fast-forward a wait instead of cutting it. - **Zoom on the action:** clicks close in time and on screen share one zoom, and the first 2.5 s stay wide. Check them, then add or move zooms with `addZooms` and `setZoom` where a result deserves its own. Hold a zoom at least 1.5 s. -- **Callouts:** one `addAnnotation` per beat, short, placed clear of the action. Text sits on the output frame and stays put when the picture zooms. Use its look for hierarchy: a large title with `animation: "typewriter"`, step callouts with `animation: "pop"`. `setAnnotation` moves or restyles one. +- **Callouts:** one `addAnnotation` per beat, short, placed clear of the action. Text sits on the output frame and stays put when the picture zooms. Use its look for hierarchy: a large title with `animation: "typewriter"`, step callouts with `animation: "pop"`. An arrow, `type: "figure"`, points at a control the text names. `setAnnotation` moves or restyles one. +- **Hide what is private:** an email address, a token or a name that shows on screen gets `addAnnotation` with `type: "blur"` over the span it is visible. Its x, y, width and height are percent of the recording's picture, not of the frame, so it stays on what it hides when a zoom moves the picture. Check it with `getFrames`. - **Undo:** `restoreCheckpoint` with the `checkpointId` takes the project back to the raw take in one step. It also puts back the take's own pointer track, undoing any `setPointerTrack` made since. - **Open in the app:** while OpenScreen is running, your edits go through it, so a project the user has open in its editor changes there live and they can undo each edit. If an edit is refused as "open in the OpenScreen editor", the app did not answer: retry once, then ask the user to close the project there or restart OpenScreen. diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.md index db9387305..d36f20614 100644 --- a/technical-documentation/architecture/ai-agent.md +++ b/technical-documentation/architecture/ai-agent.md @@ -45,7 +45,7 @@ The model never free-writes the project document. It can only call the fixed set | Tool | What it does | What it mutates | |---|---|---| -| `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert. Each zoom also reports where its camera moves sit on the ruler, outside the pill: `zoomInFromSec`, `zoomOutUntilSec` (speed-aware, from `zoomTransitions` in `src/lib/ai-edition/timeline/zoom-transitions.ts`), `transitionSec`, `cutByTrim` (`in` / `out` / `both`) when a trim cuts a move, and `cutByJunction` when a clip junction does (the compositor renders each clip on its own, so a move running past its clip's edge is lost too), so the export jumps there (#1028). A side cut by both is blamed on the cut nearest the pill, where the jump is. Chained zooms (at most 1.5 s apart on screen, no trim or clip junction between them) report the pan instead: `panToZoomId` / `panUntilSec` on the first, `panFromZoomId` on the second, in place of its zoom-out and zoom-in (`chainedPans`). | Nothing. | +| `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert, and `hideCursor` when set. Each annotation reports its type and box in the terms the writes take (`box` frame or footage, centre `x`/`y`, `width`/`height`), plus a text's look, an arrow's `figureData` or a blur's `blurData`; each audio track its gain and fades. Each zoom also reports where its camera moves sit on the ruler, outside the pill: `zoomInFromSec`, `zoomOutUntilSec` (speed-aware, from `zoomTransitions` in `src/lib/ai-edition/timeline/zoom-transitions.ts`), `transitionSec`, `cutByTrim` (`in` / `out` / `both`) when a trim cuts a move, and `cutByJunction` when a clip junction does (the compositor renders each clip on its own, so a move running past its clip's edge is lost too), so the export jumps there (#1028). A side cut by both is blamed on the cut nearest the pill, where the jump is. Chained zooms (at most 1.5 s apart on screen, no trim or clip junction between them) report the pan instead: `panToZoomId` / `panUntilSec` on the first, `panFromZoomId` on the second, in place of its zoom-out and zoom-in (`chainedPans`). | Nothing. | | `getTranscript` | Reads the transcript segments for an asset, or the primary asset, in full. On the production path a segment is one word, so a half-hour recording is a few thousand of them — there is no cap, and no per-model context budget to derive one from. | Nothing. | | `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. `visible:false` marks a hidden pointer; absent means visible, including older sidecars. Visibility changes survive downsampling even when the pointer stays still. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. | | `addTrim` | Adds one source-time cut inside a clip. Like the zoom writes and `setTrim`, it reports `cutTransitions` (zoom, side, seconds lost, `by` trim or junction) when the edit newly cuts a zoom's move in or out; a warning, never a refusal. | `timeline.trimRanges`. | @@ -55,13 +55,13 @@ The model never free-writes the project document. It can only call the fixed set | `setClipRange` | Changes a clip's source in/out points and relays clips back-to-back. | The clip range and any anchored regions clamped or removed by the shared timeline mutator. | | `moveClip` | Reorders a placed clip by naming the clip it should play before (`null` = last). Preserves every clip id, source range, trim and anchored modifier. | Timeline clip order; anchored modifiers' derived ms follow their clip. | | `replaceTimeline` | Rebuilds the primary-asset timeline from kept source-time intervals. Preserves the id, origin and label of every clip an interval reproduces exactly, carries existing trims through, and never touches another asset's trims. Refused when it would merge away, shorten or drop a clip, or when the intervals are not ascending (a reorder it cannot perform — the refusal points at `moveClip`). | Timeline clips and trim ranges. | -| `addZoom` | Adds a clip-anchored zoom over virtual timeline time. `depth` is an ordinal selecting from `ZOOM_DEPTH_SCALES` (1.25×–5.0×, non-linear); the result reports the resulting `renderedScale`. When the pointer track can be read for the footage under the span it also reports `cursorAnchor`: the `focus` the call used (default included) beside the median recorded pointer position over the span the zoom LANDED on, with `spread`, `offset` and the sample count. The source windows come from the stored fragments' own `sourceStartSec`/`sourceEndSec`, so a clamped or split zoom is described by the windows it really occupies; trimmed instants are left out; and the field is OMITTED — never guessed, never turned into an absence of data — when no track covers that footage. | `zoomRanges`. | +| `addZoom` | Adds a clip-anchored zoom over virtual timeline time. `depth` is an ordinal selecting from `ZOOM_DEPTH_SCALES` (1.25×–5.0×, non-linear); the result reports the resulting `renderedScale`. It also takes the inspector's `focusMode`, `hideCursor` and `customScale`, under the same rules as `setZoom`. When the pointer track can be read for the footage under the span it also reports `cursorAnchor`: the `focus` the call used (default included) beside the median recorded pointer position over the span the zoom LANDED on, with `spread`, `offset` and the sample count. The source windows come from the stored fragments' own `sourceStartSec`/`sourceEndSec`, so a clamped or split zoom is described by the windows it really occupies; trimmed instants are left out; and the field is OMITTED — never guessed, never turned into an absence of data — when no track covers that footage. | `zoomRanges`. | | `addZooms` | Adds many zooms in one call, replaying `addZoom` per entry, with the same per-entry refusal and reporting contract as `addTrims`. Each applied entry carries its own `cursorAnchor`. | `zoomRanges`. | -| `setZoom` | Moves, resizes, or restyles a zoom pill. Changing `depth` clears any `customScale` on that pill — otherwise the write is a no-op at render — and says so in the result. Reports `cursorAnchor` against the zoom's EFFECTIVE focus, read back off the document, so a span-only edit still learns what its unchanged focus now looks at. | The clip-anchored `zoomRanges` fragments represented by that pill. | +| `setZoom` | Moves, resizes, or restyles a zoom pill: level, focus, `focusMode`, `hideCursor`, 3D camera. Changing `depth` clears any `customScale` on that pill — otherwise the write is a no-op at render — and says so in the result. A `customScale` is written as the inspector writes a typed level (`zoomLevelFor`: to the hundredth, a table level as its depth) and refused past the clip's `zoomScaleLimit`; `depth` with `customScale` is refused. Reports `cursorAnchor` against the zoom's EFFECTIVE focus, read back off the document, so a span-only edit still learns what its unchanged focus now looks at. | The clip-anchored `zoomRanges` fragments represented by that pill. | | `addSpeed` | Adds a clip-anchored speed region over virtual timeline time. | `legacyEditor.speedRegions`. | | `setSpeed` | Moves, resizes, or changes an existing speed pill. | The corresponding `legacyEditor.speedRegions` fragments. | -| `addAnnotation` | Adds a positioned text annotation over virtual timeline time. | `annotations`. | -| `setAnnotation` | Moves, resizes, or changes an annotation's text. | The corresponding clip-anchored `annotations` fragments. | +| `addAnnotation` | Adds a text, arrow (`figure`) or privacy `blur` annotation over virtual timeline time. An arrow or a blur is the editor's text switched through `convertAnnotationKind`, then given its box and its `figureData` / `blurData` (validated by `figureDataSchema` / `blurDataSchema`). Images are left out: they need a file. A freehand blur is refused, as the inspector no longer offers it. | `annotations`. | +| `setAnnotation` | Moves, resizes, retypes or restyles an annotation. A type change goes through `convertAnnotationKind` with the footage under each fragment (`annotationFootageRect`), as the inspector's type switch does; `figureData` / `blurData` merge only the keys sent. Arguments of another kind (width on a text, blurData on an arrow) are refused. | The corresponding clip-anchored `annotations` fragments. | | `addCameraFullscreen` | Adds a camera-fullscreen region over virtual timeline time; refused when no clip under the span comes from an asset with a linked `cameraTrack`, since such a region can only render nothing. | `legacyEditor.cameraFullscreenRegions`. | | `setCameraFullscreen` | Moves or resizes a camera-fullscreen pill, under the same camera requirement as `addCameraFullscreen`. | The corresponding `legacyEditor.cameraFullscreenRegions` fragments. | | `removeTrim` | Deletes a trim so its source span plays and exports again. | `timeline.trimRanges`. | diff --git a/website/docs/ai-editing.md b/website/docs/ai-editing.md index 4015709a8..04984cfca 100644 --- a/website/docs/ai-editing.md +++ b/website/docs/ai-editing.md @@ -41,7 +41,7 @@ The ChatGPT and GitHub Copilot sign-in options were **removed in 1.8.0**. They w ## Using the agent -Describe the edit in plain language — "cut the dead air in the intro", "zoom in when I open the terminal". The agent works through real, undoable timeline operations, not a re-render: it can add and adjust trims, zooms, speed regions, annotations and Full Camera segments, edit clip in/out points, reorder or remove clips, and read the transcript to find what you're referring to. +Describe the edit in plain language — "cut the dead air in the intro", "zoom in when I open the terminal". The agent works through real, undoable timeline operations, not a re-render: it can add and adjust trims, zooms, speed regions, annotations (text, arrows, and blurs that hide private details on screen), imported audio tracks with their fades, and Full Camera segments, edit clip in/out points, reorder or remove clips, and read the transcript to find what you're referring to. The panel around it: diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md index 3c700938d..1d435b948 100644 --- a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Die Anmeldeoptionen für ChatGPT und GitHub Copilot wurden **in 1.8.0 entfernt** ## Den Agenten nutzen {#using-the-agent} -Beschreibe die Änderung in Alltagssprache, etwa „schneide die Stille im Intro heraus“ oder „zoome hinein, wenn ich das Terminal öffne“. Der Agent arbeitet mit echten, rückgängig machbaren Operationen auf der Zeitleiste, nicht mit einem neuen Rendering: Er kann Schnitte, Zooms, Geschwindigkeitsbereiche, Annotationen und Full-Camera-Segmente hinzufügen und anpassen, Start- und Endpunkte von Clips ändern, Clips umsortieren oder entfernen und das Transkript lesen, um herauszufinden, was du meinst. +Beschreibe die Änderung in Alltagssprache, etwa „schneide die Stille im Intro heraus“ oder „zoome hinein, wenn ich das Terminal öffne“. Der Agent arbeitet mit echten, rückgängig machbaren Operationen auf der Zeitleiste, nicht mit einem neuen Rendering: Er kann Schnitte, Zooms, Geschwindigkeitsbereiche, Annotationen (Text, Pfeile und Unschärfen, die private Details auf dem Bildschirm verbergen), importierte Audiospuren samt Ein- und Ausblendung und Full-Camera-Segmente hinzufügen und anpassen, Start- und Endpunkte von Clips ändern, Clips umsortieren oder entfernen und das Transkript lesen, um herauszufinden, was du meinst. Der Bereich drumherum: diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md index ed57e68ed..658fa2a42 100644 --- a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Las opciones de inicio de sesión con ChatGPT y GitHub Copilot se **eliminaron e ## Usar el agente {#using-the-agent} -Describe la edición con tus propias palabras: "corta el tiempo muerto de la introducción", "haz zoom cuando abro la terminal". El agente trabaja con operaciones reales de la línea de tiempo, que se pueden deshacer, y no con un nuevo renderizado: puede agregar y ajustar recortes, zooms, regiones de velocidad, anotaciones y segmentos de cámara a pantalla completa, editar los puntos de entrada y salida de los clips, reordenar o quitar clips, y leer la transcripción para encontrar aquello a lo que te refieres. +Describe la edición con tus propias palabras: "corta el tiempo muerto de la introducción", "haz zoom cuando abro la terminal". El agente trabaja con operaciones reales de la línea de tiempo, que se pueden deshacer, y no con un nuevo renderizado: puede agregar y ajustar recortes, zooms, regiones de velocidad, anotaciones (texto, flechas y desenfoques que ocultan datos privados en pantalla), pistas de audio importadas con sus fundidos y segmentos de cámara a pantalla completa, editar los puntos de entrada y salida de los clips, reordenar o quitar clips, y leer la transcripción para encontrar aquello a lo que te refieres. El panel que lo rodea: diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md index 0c1cfa146..ccfbd7398 100644 --- a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Les options de connexion ChatGPT et GitHub Copilot ont été **supprimées dans ## Utiliser l'agent {#using-the-agent} -Décrivez la modification en langage courant : « coupe les temps morts de l'intro », « zoome quand j'ouvre le terminal ». L'agent travaille avec de vraies opérations de timeline annulables, pas avec un nouveau rendu : il peut ajouter et ajuster des coupes, des zooms, des régions de vitesse, des annotations et des segments Caméra plein écran, modifier les points d'entrée et de sortie des clips, réordonner ou retirer des clips, et lire la transcription pour trouver ce dont vous parlez. +Décrivez la modification en langage courant : « coupe les temps morts de l'intro », « zoome quand j'ouvre le terminal ». L'agent travaille avec de vraies opérations de timeline annulables, pas avec un nouveau rendu : il peut ajouter et ajuster des coupes, des zooms, des régions de vitesse, des annotations (texte, flèches et flous qui masquent une information privée à l'écran), des pistes audio importées avec leurs fondus, et des segments Caméra plein écran, modifier les points d'entrée et de sortie des clips, réordonner ou retirer des clips, et lire la transcription pour trouver ce dont vous parlez. Le panneau qui l'entoure : diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md index d2be7ea54..f885f150b 100644 --- a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT と GitHub Copilot のサインインは、**1.8.0 で削除されまし ## エージェントを使う {#using-the-agent} -行いたい編集を、普段の言葉で伝えます（「イントロの無音部分をカットして」「ターミナルを開いたところでズームして」など）。エージェントは再レンダリングではなく、元に戻せる実際のタイムライン操作で作業します。トリム、ズーム、再生速度の範囲、注釈、フルスクリーンカメラのセグメントの追加と調整、クリップのイン点とアウト点の編集、クリップの並べ替えと削除ができ、文字起こしを読んで、あなたが指している箇所を見つけます。 +行いたい編集を、普段の言葉で伝えます（「イントロの無音部分をカットして」「ターミナルを開いたところでズームして」など）。エージェントは再レンダリングではなく、元に戻せる実際のタイムライン操作で作業します。トリム、ズーム、再生速度の範囲、注釈（テキスト、矢印、画面上の個人情報を隠すぼかし）、取り込んだオーディオトラックとそのフェード、フルスクリーンカメラのセグメントの追加と調整、クリップのイン点とアウト点の編集、クリップの並べ替えと削除ができ、文字起こしを読んで、あなたが指している箇所を見つけます。 エージェントを囲むパネル： diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md index 22086702a..2eac6c45a 100644 --- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ As opções de login do ChatGPT e do GitHub Copilot foram **removidas na 1.8.0** ## Como usar o agente {#using-the-agent} -Descreva a edição em linguagem natural — "corte o tempo morto da introdução", "dê zoom quando eu abrir o terminal". O agente trabalha com operações reais da linha do tempo, que podem ser desfeitas, e não com uma nova renderização: ele pode adicionar e ajustar recortes, zooms, regiões de velocidade, anotações e segmentos de Câmera em Tela Cheia, editar os pontos de entrada/saída dos clipes, reordenar ou remover clipes e ler a transcrição para encontrar o trecho a que você se refere. +Descreva a edição em linguagem natural — "corte o tempo morto da introdução", "dê zoom quando eu abrir o terminal". O agente trabalha com operações reais da linha do tempo, que podem ser desfeitas, e não com uma nova renderização: ele pode adicionar e ajustar recortes, zooms, regiões de velocidade, anotações (texto, setas e desfoques que escondem dados privados na tela), faixas de áudio importadas com seus fades e segmentos de Câmera em Tela Cheia, editar os pontos de entrada/saída dos clipes, reordenar ou remover clipes e ler a transcrição para encontrar o trecho a que você se refere. O painel em volta dele: diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md index 2a110245d..97192274a 100644 --- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT 和 GitHub Copilot 登录选项已**在 1.8.0 中移除**。它们的工 ## 使用 AI 代理 {#using-the-agent} -用日常语言描述你想要的编辑，例如“剪掉开头的空白”“我打开终端时放大画面”。AI 代理通过真实、可撤销的时间轴操作来完成编辑，而不是重新渲染：它可以添加和调整删减区间、缩放、速度区间、标注和全屏摄像头区间，编辑片段的入点/出点，重新排序或移除片段，还会读取转录文本，找到你所指的内容。 +用日常语言描述你想要的编辑，例如“剪掉开头的空白”“我打开终端时放大画面”。AI 代理通过真实、可撤销的时间轴操作来完成编辑，而不是重新渲染：它可以添加和调整删减区间、缩放、速度区间、标注（文字、箭头，以及遮住屏幕上隐私信息的模糊）、导入的音轨及其淡入淡出和全屏摄像头区间，编辑片段的入点/出点，重新排序或移除片段，还会读取转录文本，找到你所指的内容。 它周围的面板包括： diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md index 0a425d6dc..b48e12f5d 100644 --- a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT 與 GitHub Copilot 的登入選項已**在 1.8.0 中移除**。它們的 ## 使用代理 {#using-the-agent} -用一般的語言描述你要的剪輯，例如「剪掉開頭的空檔」、「我打開終端機時放大畫面」。代理會透過真實、可復原的時間軸操作來完成工作，而不是重新算繪：它可以新增與調整剪輯、縮放、速度區域、標註與全螢幕攝影機片段，編輯片段的入點與出點，重新排序或移除片段，也會讀取逐字稿來找出你指的是哪一段。 +用一般的語言描述你要的剪輯，例如「剪掉開頭的空檔」、「我打開終端機時放大畫面」。代理會透過真實、可復原的時間軸操作來完成工作，而不是重新算繪：它可以新增與調整剪輯、縮放、速度區域、標註（文字、箭頭，以及遮住畫面上隱私資訊的模糊）、匯入的音軌及其淡入淡出與全螢幕攝影機片段，編輯片段的入點與出點，重新排序或移除片段，也會讀取逐字稿來找出你指的是哪一段。 它周圍的面板包括：