diff --git a/CREDITS.md b/CREDITS.md index fe6177e0f..129fe1ec7 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -22,3 +22,9 @@ Remotion. Thanks also to the authors and maintainers of the open-source projects HyperFrames builds on, including Puppeteer, FFmpeg, GSAP, Hono, and the broader Node.js ecosystem. + +## Third-party licenses + +- **[mediabunny](https://github.com/nicoch/mediabunny)** — media toolkit used + in the studio for fast metadata extraction from file headers. Licensed under + the [Mozilla Public License 2.0 (MPL-2.0)](https://mozilla.org/MPL/2.0/). diff --git a/bun.lock b/bun.lock index 84d73f6db..8c86b1b31 100644 --- a/bun.lock +++ b/bun.lock @@ -22,7 +22,7 @@ }, "packages/aws-lambda": { "name": "@hyperframes/aws-lambda", - "version": "0.6.22", + "version": "0.6.27", "dependencies": { "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/client-sfn": "^3.700.0", @@ -54,7 +54,7 @@ }, "packages/cli": { "name": "@hyperframes/cli", - "version": "0.6.22", + "version": "0.6.27", "bin": { "hyperframes": "./dist/cli.js", }, @@ -97,7 +97,7 @@ }, "packages/core": { "name": "@hyperframes/core", - "version": "0.6.22", + "version": "0.6.27", "dependencies": { "@chenglou/pretext": "^0.0.5", "postcss": "^8.5.8", @@ -124,7 +124,7 @@ }, "packages/engine": { "name": "@hyperframes/engine", - "version": "0.6.22", + "version": "0.6.27", "dependencies": { "@hono/node-server": "^1.13.0", "@hyperframes/core": "workspace:^", @@ -142,7 +142,7 @@ }, "packages/player": { "name": "@hyperframes/player", - "version": "0.6.22", + "version": "0.6.27", "devDependencies": { "@types/bun": "^1.1.0", "gsap": "^3.12.5", @@ -154,7 +154,7 @@ }, "packages/producer": { "name": "@hyperframes/producer", - "version": "0.6.22", + "version": "0.6.27", "dependencies": { "@fontsource/archivo-black": "^5.2.8", "@fontsource/eb-garamond": "^5.2.7", @@ -194,7 +194,7 @@ }, "packages/shader-transitions": { "name": "@hyperframes/shader-transitions", - "version": "0.6.22", + "version": "0.6.27", "dependencies": { "html2canvas": "^1.4.1", }, @@ -206,7 +206,7 @@ }, "packages/studio": { "name": "@hyperframes/studio", - "version": "0.6.22", + "version": "0.6.27", "dependencies": { "@codemirror/autocomplete": "^6.20.1", "@codemirror/commands": "^6.10.3", @@ -221,6 +221,7 @@ "@hyperframes/core": "workspace:*", "@hyperframes/player": "workspace:*", "@phosphor-icons/react": "^2.1.10", + "mediabunny": "^1.45.3", }, "devDependencies": { "@hyperframes/producer": "workspace:*", @@ -962,6 +963,10 @@ "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + "@types/dom-mediacapture-transform": ["@types/dom-mediacapture-transform@0.1.11", "", { "dependencies": { "@types/dom-webcodecs": "*" } }, "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ=="], + + "@types/dom-webcodecs": ["@types/dom-webcodecs@0.1.13", "", {}, "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ=="], + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], "@types/jsdom": ["@types/jsdom@28.0.2", "", { "dependencies": { "@types/node": "*", "@types/tough-cookie": "*", "parse5": "^8.0.0", "undici-types": "^7.21.0" } }, "sha512-zZYItekplnGirFhVDrcB0+103TMakXfKfIp7uECxaFzFG3Ws5kYQSwVb1d4pQfJMMjQda6pfuZxueAv9CMiJbw=="], @@ -1504,6 +1509,8 @@ "mdn-data": ["mdn-data@2.27.1", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="], + "mediabunny": ["mediabunny@1.45.3", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-GUCPYjR+5olLM7DRmupCXCmZkkSrKHVl1gyW2RztpObqLfrix19kWGf/9WgWzDW0g49DvfGXpl+zfArCx5HDMQ=="], + "meow": ["meow@13.2.0", "", {}, "sha512-pxQJQzB6djGPXh08dacEloMFopsOqGVRKFPYvPOt9XDZ1HasbgDZA74CJGreSU4G3Ak7EFJGoiH2auq+yXISgA=="], "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], diff --git a/packages/core/src/runtime/mediaPreloader.test.ts b/packages/core/src/runtime/mediaPreloader.test.ts index 792ed1fb7..0226c04c7 100644 --- a/packages/core/src/runtime/mediaPreloader.test.ts +++ b/packages/core/src/runtime/mediaPreloader.test.ts @@ -43,6 +43,19 @@ function setupDOM(elements: HTMLMediaElement[]): void { }) as typeof document.querySelectorAll; } +function createTestFixture( + count: number, + options?: Parameters[0], +) { + const elements = Array.from({ length: count }, (_, i) => + mockMediaElement({ start: String(i * 5), duration: "5" }), + ); + setupDOM(elements); + const manager = createMediaPreloadManager(options); + manager.refresh(); + return { elements, manager }; +} + describe("createMediaPreloadManager", () => { let elements: HTMLMediaElement[]; @@ -50,7 +63,7 @@ describe("createMediaPreloadManager", () => { elements = []; }); - it("is not lazy when fewer than 6 media elements", () => { + it("is not lazy when fewer than 3 media elements", () => { elements = [ mockMediaElement({ start: "0", duration: "5" }), mockMediaElement({ start: "5", duration: "5" }), @@ -63,274 +76,151 @@ describe("createMediaPreloadManager", () => { expect(manager.isLazy()).toBe(false); }); - it("activates lazy mode at exactly LAZY_THRESHOLD (6 elements)", () => { - elements = Array.from({ length: 6 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - + it("activates lazy mode at exactly LAZY_THRESHOLD (3 elements)", () => { + const { manager } = createTestFixture(3); expect(manager.isLazy()).toBe(true); }); - it("is not lazy with 5 elements (below threshold)", () => { - elements = Array.from({ length: 5 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - + it("is not lazy with 2 elements (below threshold)", () => { + const { manager } = createTestFixture(2); expect(manager.isLazy()).toBe(false); }); it("activates lazy mode with 8 media elements", () => { - elements = Array.from({ length: 8 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - + const { manager } = createTestFixture(8); expect(manager.isLazy()).toBe(true); }); - it("sync promotes clips in the lookahead window", () => { - elements = Array.from({ length: 8 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), + it("activates lazy mode for 4-5 clip compositions without spurious eviction", () => { + const f = createTestFixture(4); + expect(f.manager.isLazy()).toBe(true); + for (const el of f.elements) el.preload = "metadata"; + f.manager.sync(0); + const promoted = f.elements.filter((el) => el.preload === "auto").length; + expect(promoted).toBeGreaterThanOrEqual(2); + expect(promoted).toBeLessThanOrEqual(4); + f.manager.sync(0); + const evicted = f.elements.filter( + (el) => + el.preload === "metadata" && (el.load as ReturnType).mock.calls.length > 1, ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - - for (const el of elements) { - el.preload = "metadata"; - } - - manager.sync(0); + expect(evicted.length).toBe(0); + }); - expect(elements[0].preload).toBe("auto"); - expect(elements[1].preload).toBe("auto"); - expect(elements[7].preload).toBe("metadata"); + it("sync promotes clips in the lookahead window", () => { + const f = createTestFixture(8); + for (const el of f.elements) el.preload = "metadata"; + f.manager.sync(0); + expect(f.elements[0].preload).toBe("auto"); + expect(f.elements[1].preload).toBe("auto"); + expect(f.elements[7].preload).toBe("metadata"); }); it("preloadAroundTime promotes clips near seek target", () => { - elements = Array.from({ length: 10 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - - for (const el of elements) { - el.preload = "metadata"; - } - - manager.preloadAroundTime(30); - - expect(elements[6].preload).toBe("auto"); - expect(elements[7].preload).toBe("auto"); - expect(elements[0].preload).toBe("metadata"); + const f = createTestFixture(10); + for (const el of f.elements) el.preload = "metadata"; + f.manager.preloadAroundTime(30); + expect(f.elements[6].preload).toBe("auto"); + expect(f.elements[7].preload).toBe("auto"); + expect(f.elements[0].preload).toBe("metadata"); }); it("sync is a no-op when not lazy", () => { - elements = [ - mockMediaElement({ start: "0", duration: "5" }), - mockMediaElement({ start: "5", duration: "5" }), - ]; - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - manager.sync(0); - - expect(manager.isLazy()).toBe(false); + const f = createTestFixture(2); + f.manager.sync(0); + expect(f.manager.isLazy()).toBe(false); }); it("guarantees at least LOOKAHEAD_MIN_CLIPS are promoted", () => { + // Use 20s spacing so only 1 clip falls in the 10s lookahead window elements = Array.from({ length: 8 }, (_, i) => mockMediaElement({ start: String(i * 20), duration: "5" }), ); setupDOM(elements); - const manager = createMediaPreloadManager(); manager.refresh(); - - for (const el of elements) { - el.preload = "metadata"; - } - + for (const el of elements) el.preload = "metadata"; manager.sync(0); - - const promotedCount = elements.filter((el) => el.preload === "auto").length; - expect(promotedCount).toBeGreaterThanOrEqual(2); + expect(elements.filter((el) => el.preload === "auto").length).toBeGreaterThanOrEqual(2); }); it("evicts clips when scrubbing away from them", () => { - elements = Array.from({ length: 10 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - - for (const el of elements) { - el.preload = "metadata"; - } - - // Promote clips around t=0 - manager.sync(0); - expect(elements[0].preload).toBe("auto"); - expect(elements[1].preload).toBe("auto"); - - // Scrub to t=40 — clips 0,1 should be evicted - manager.sync(40); - expect(elements[0].preload).toBe("metadata"); - expect(elements[0].src).toBe(""); - expect(elements[8].preload).toBe("auto"); + const f = createTestFixture(10); + for (const el of f.elements) el.preload = "metadata"; + f.manager.sync(0); + expect(f.elements[0].preload).toBe("auto"); + f.manager.sync(40); + expect(f.elements[0].preload).toBe("metadata"); + expect(f.elements[0].src).toBe(""); + expect(f.elements[8].preload).toBe("auto"); }); it("restores src when re-promoting a previously evicted clip", () => { - elements = Array.from({ length: 10 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - - for (const el of elements) { - el.preload = "metadata"; - } - - const originalSrc0 = elements[0].src; - - // Promote at t=0, scrub away, scrub back - manager.sync(0); - manager.sync(40); - expect(elements[0].src).toBe(""); - - manager.sync(0); - expect(elements[0].src).toBe(originalSrc0); - expect(elements[0].preload).toBe("auto"); + const f = createTestFixture(10); + for (const el of f.elements) el.preload = "metadata"; + const originalSrc0 = f.elements[0].src; + f.manager.sync(0); + f.manager.sync(40); + expect(f.elements[0].src).toBe(""); + f.manager.sync(0); + expect(f.elements[0].src).toBe(originalSrc0); + expect(f.elements[0].preload).toBe("auto"); }); it("does not exceed MAX_PROMOTED (5) clips", () => { - // 10 clips, each 5s long, spaced 5s apart - elements = Array.from({ length: 10 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - - for (const el of elements) { - el.preload = "metadata"; - } - - // Sync at t=0 — window covers clips 0,1,2 (0-15s lookahead) - manager.sync(0); - const promotedAfterFirst = elements.filter((el) => el.preload === "auto").length; - expect(promotedAfterFirst).toBeLessThanOrEqual(5); - - // Sync at different position — should evict old ones - manager.sync(25); - const totalPromoted = elements.filter((el) => el.preload === "auto").length; - expect(totalPromoted).toBeLessThanOrEqual(5); + const f = createTestFixture(10); + for (const el of f.elements) el.preload = "metadata"; + f.manager.sync(0); + expect(f.elements.filter((el) => el.preload === "auto").length).toBeLessThanOrEqual(5); + f.manager.sync(25); + expect(f.elements.filter((el) => el.preload === "auto").length).toBeLessThanOrEqual(5); }); it("calls load() when evicting to release buffers", () => { - elements = Array.from({ length: 10 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), + const f = createTestFixture(10); + for (const el of f.elements) el.preload = "metadata"; + f.manager.sync(0); + const loadCallsBefore = (f.elements[0].load as ReturnType).mock.calls.length; + f.manager.sync(40); + expect((f.elements[0].load as ReturnType).mock.calls.length).toBeGreaterThan( + loadCallsBefore, ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); - - for (const el of elements) { - el.preload = "metadata"; - } - - manager.sync(0); - const loadCallsBefore = (elements[0].load as ReturnType).mock.calls.length; - - // Scrub away — eviction should call load() to release buffers - manager.sync(40); - const loadCallsAfter = (elements[0].load as ReturnType).mock.calls.length; - expect(loadCallsAfter).toBeGreaterThan(loadCallsBefore); }); it("isLazy reports true with 6+ clips so caller can gate render-mode bypass", () => { - elements = Array.from({ length: 6 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - - const manager = createMediaPreloadManager(); - manager.refresh(); + const { manager } = createTestFixture(6); expect(manager.isLazy()).toBe(true); }); it("calls onActivation when lazy mode activates", () => { - elements = Array.from({ length: 8 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - const onActivation = vi.fn(); - const manager = createMediaPreloadManager({ onActivation }); - manager.refresh(); - + createTestFixture(8, { onActivation }); expect(onActivation).toHaveBeenCalledOnce(); expect(onActivation).toHaveBeenCalledWith(8); }); it("does not call onActivation below threshold", () => { - elements = [ - mockMediaElement({ start: "0", duration: "5" }), - mockMediaElement({ start: "5", duration: "5" }), - ]; - setupDOM(elements); - const onActivation = vi.fn(); - const manager = createMediaPreloadManager({ onActivation }); - manager.refresh(); - + createTestFixture(2, { onActivation }); expect(onActivation).not.toHaveBeenCalled(); }); it("calls onActivation only once across multiple refreshes", () => { - elements = Array.from({ length: 8 }, (_, i) => - mockMediaElement({ start: String(i * 5), duration: "5" }), - ); - setupDOM(elements); - const onActivation = vi.fn(); - const manager = createMediaPreloadManager({ onActivation }); + const { manager } = createTestFixture(8, { onActivation }); manager.refresh(); manager.refresh(); - manager.refresh(); - expect(onActivation).toHaveBeenCalledOnce(); }); it("respects window.__HF_LAZY_PRELOAD_THRESHOLD override", () => { - elements = Array.from({ length: 4 }, (_, i) => + elements = Array.from({ length: 2 }, (_, i) => mockMediaElement({ start: String(i * 5), duration: "5" }), ); setupDOM(elements); - // 4 elements is below the default threshold (6) but at our custom one - (window as Record).__HF_LAZY_PRELOAD_THRESHOLD = 4; + // 2 elements is below the default threshold (3) but at our custom one + (window as Record).__HF_LAZY_PRELOAD_THRESHOLD = 2; const manager = createMediaPreloadManager(); manager.refresh(); @@ -342,7 +232,7 @@ describe("createMediaPreloadManager", () => { }); it("falls back to default threshold when __HF_LAZY_PRELOAD_THRESHOLD is not set", () => { - elements = Array.from({ length: 4 }, (_, i) => + elements = Array.from({ length: 2 }, (_, i) => mockMediaElement({ start: String(i * 5), duration: "5" }), ); setupDOM(elements); diff --git a/packages/core/src/runtime/mediaPreloader.ts b/packages/core/src/runtime/mediaPreloader.ts index 6e4d2b4f6..86345061c 100644 --- a/packages/core/src/runtime/mediaPreloader.ts +++ b/packages/core/src/runtime/mediaPreloader.ts @@ -1,17 +1,18 @@ import { refreshRuntimeMediaCache, type RuntimeMediaClip } from "./media"; -// Compositions with fewer than 6 timed clips rarely exceed browser memory -// limits during eager preload. The threshold avoids preload management -// overhead for typical compositions while catching the heavy-media case -// (e.g., 20 clips / 6GB reported in heygen-com/hyperframes#729). -const LAZY_THRESHOLD = 6; +// Start lazy preload management at 3 clips to keep memory pressure low from +// the start. The previous threshold of 6 let medium compositions (4–5 heavy +// videos) saturate browser memory before the preloader kicked in. +const LAZY_THRESHOLD = 3; const LOOKAHEAD_SECONDS = 10; +const LOOKBEHIND_SECONDS = 3; const LOOKAHEAD_MIN_CLIPS = 2; -// Cap on simultaneously promoted (buffered) clips. When the lookahead window -// contains more clips than this (e.g., many short clips), all window clips -// stay promoted — the cap is defense-in-depth, not a hard ceiling. The primary -// memory bound comes from window-based eviction in syncWindow(). -const MAX_PROMOTED = 5; +// Adaptive cap: base of 4 for small sets, clamped to 6 for larger ones. +// The window-based eviction in syncWindow() is the primary memory bound; +// this cap is defense-in-depth for compositions with many short clips +// packed into the lookahead window. +const MAX_PROMOTED_BASE = 4; +const MAX_PROMOTED_CEIL = 6; export interface MediaPreloadManager { refresh(): void; @@ -91,23 +92,23 @@ export function createMediaPreloadManager(options?: { windowEls.add(clip.el); } - // Evict clips no longer in window, oldest first for (const clip of clips) { if (promoted.has(clip.el) && !windowEls.has(clip.el)) { evictClip(clip); } } - // If still over budget after removing out-of-window clips, - // evict the oldest promoted that isn't in the current window - while (promotionOrder.length > MAX_PROMOTED) { + const maxPromoted = Math.min( + MAX_PROMOTED_CEIL, + MAX_PROMOTED_BASE + Math.floor(clips.length / 10), + ); + while (promotionOrder.length > maxPromoted) { const oldest = promotionOrder[0]; - if (windowEls.has(oldest)) break; // don't evict something currently needed + if (windowEls.has(oldest)) break; const clip = clips.find((c) => c.el === oldest); if (clip) { evictClip(clip); } else { - // Element no longer in clips list, just remove from tracking promoted.delete(oldest); promotionOrder.shift(); } @@ -115,13 +116,15 @@ export function createMediaPreloadManager(options?: { } function getClipsInWindow(timeSeconds: number): Set { + const windowStart = timeSeconds - LOOKBEHIND_SECONDS; const windowEnd = timeSeconds + LOOKAHEAD_SECONDS; const inWindow = new Set(); for (const clip of clips) { const active = timeSeconds >= clip.start && timeSeconds < clip.end; const inLookahead = clip.start >= timeSeconds && clip.start <= windowEnd; - if (active || inLookahead) { + const inLookbehind = clip.end > windowStart && clip.end <= timeSeconds; + if (active || inLookahead || inLookbehind) { inWindow.add(clip); } } diff --git a/packages/studio/package.json b/packages/studio/package.json index 6279605d7..0e143a1d2 100644 --- a/packages/studio/package.json +++ b/packages/studio/package.json @@ -38,7 +38,8 @@ "@codemirror/view": "6.40.0", "@hyperframes/core": "workspace:*", "@hyperframes/player": "workspace:*", - "@phosphor-icons/react": "^2.1.10" + "@phosphor-icons/react": "^2.1.10", + "mediabunny": "^1.45.3" }, "devDependencies": { "@hyperframes/producer": "workspace:*", diff --git a/packages/studio/src/App.tsx b/packages/studio/src/App.tsx index 7a0360b60..ed792040d 100644 --- a/packages/studio/src/App.tsx +++ b/packages/studio/src/App.tsx @@ -8,7 +8,7 @@ import { useCaptionSync } from "./captions/hooks/useCaptionSync"; import { usePersistentEditHistory } from "./hooks/usePersistentEditHistory"; import { usePanelLayout } from "./hooks/usePanelLayout"; import { useFileManager } from "./hooks/useFileManager"; -import { useManifestPersistence } from "./hooks/useManifestPersistence"; +import { usePreviewPersistence } from "./hooks/usePreviewPersistence"; import { useTimelineEditing } from "./hooks/useTimelineEditing"; import { addBlockToProject } from "./utils/blockInstaller"; import type { BlockParam } from "@hyperframes/core/registry"; @@ -117,12 +117,9 @@ export function StudioApp() { }); const editHistory = usePersistentEditHistory({ projectId }); const domEditSaveTimestampRef = useRef(0); + const pendingTimelineEditPathRef = useRef(new Set()); const reloadPreview = useCallback(() => { - try { - previewIframeRef.current?.contentWindow?.location.reload(); - } catch { - setRefreshKey((k) => k + 1); - } + setRefreshKey((k) => k + 1); }, []); const fileManager = useFileManager({ @@ -145,7 +142,7 @@ export function StudioApp() { setActiveCompPathHydrated(true); }, [activeCompPathHydrated, fileManager.fileTree, fileManager.fileTreeLoaded]); - const manifestPersistence = useManifestPersistence({ + const previewPersistence = usePreviewPersistence({ projectId, showToast, readOptionalProjectFile: fileManager.readOptionalProjectFile, @@ -155,6 +152,7 @@ export function StudioApp() { activeCompPathRef, domEditSaveTimestampRef, reloadPreview: () => setRefreshKey((k) => k + 1), + pendingTimelineEditPathRef, }); const timelineEditing = useTimelineEditing({ @@ -166,6 +164,8 @@ export function StudioApp() { recordEdit: editHistory.recordEdit, domEditSaveTimestampRef, reloadPreview, + previewIframeRef, + pendingTimelineEditPathRef, uploadProjectFiles: fileManager.uploadProjectFiles, }); @@ -274,8 +274,8 @@ export function StudioApp() { writeProjectFile: fileManager.writeProjectFile, domEditSaveTimestampRef, showToast, - syncHistoryPreviewAfterApply: manifestPersistence.syncHistoryPreviewAfterApply, - waitForPendingDomEditSaves: manifestPersistence.waitForPendingDomEditSaves, + syncHistoryPreviewAfterApply: previewPersistence.syncHistoryPreviewAfterApply, + waitForPendingDomEditSaves: previewPersistence.waitForPendingDomEditSaves, leftSidebarRef, handleCopy, handlePaste, @@ -297,7 +297,7 @@ export function StudioApp() { setRightPanelTab: panelLayout.setRightPanelTab, showToast, refreshPreviewDocumentVersion, - queueDomEditSave: manifestPersistence.queueDomEditSave, + queueDomEditSave: previewPersistence.queueDomEditSave, readProjectFile: fileManager.readProjectFile, writeProjectFile: fileManager.writeProjectFile, domEditSaveTimestampRef, @@ -309,7 +309,7 @@ export function StudioApp() { previewIframe, refreshKey, rightPanelTab: panelLayout.rightPanelTab, - applyStudioManualEditsToPreviewRef: manifestPersistence.applyStudioManualEditsToPreviewRef, + applyStudioManualEditsToPreviewRef: previewPersistence.applyStudioManualEditsToPreviewRef, syncPreviewHistoryHotkey: appHotkeys.syncPreviewHistoryHotkey, reloadPreview, setRefreshKey, @@ -345,7 +345,7 @@ export function StudioApp() { projectId, activeCompPath, showToast, - waitForPendingDomEditSaves: manifestPersistence.waitForPendingDomEditSaves, + waitForPendingDomEditSaves: previewPersistence.waitForPendingDomEditSaves, }); const { consoleErrors, @@ -453,7 +453,7 @@ export function StudioApp() { startRender: renderQueue.startRender as (options: unknown) => Promise, }, compositionDimensions, - waitForPendingDomEditSaves: manifestPersistence.waitForPendingDomEditSaves, + waitForPendingDomEditSaves: previewPersistence.waitForPendingDomEditSaves, handlePreviewIframeRef, refreshPreviewDocumentVersion, timelineVisible, diff --git a/packages/studio/src/hooks/useManifestPersistence.ts b/packages/studio/src/hooks/usePreviewPersistence.ts similarity index 90% rename from packages/studio/src/hooks/useManifestPersistence.ts rename to packages/studio/src/hooks/usePreviewPersistence.ts index d5c859f1f..eab2d755d 100644 --- a/packages/studio/src/hooks/useManifestPersistence.ts +++ b/packages/studio/src/hooks/usePreviewPersistence.ts @@ -17,7 +17,7 @@ interface RecordEditInput { files: Record; } -interface UseManifestPersistenceParams { +interface UsePreviewPersistenceParams { projectId: string | null; showToast: (message: string, tone?: "error" | "info") => void; readOptionalProjectFile: (path: string) => Promise; @@ -26,15 +26,18 @@ interface UseManifestPersistenceParams { previewIframeRef: React.MutableRefObject; activeCompPathRef: React.MutableRefObject; /** Shared timestamp ref — written by any studio save (code tab, timeline, DOM edits). - * Used to suppress SSE echoes so we don't double-reload after our own saves. */ + * Used to suppress file-change echoes so we don't reload after our own saves. */ domEditSaveTimestampRef: React.MutableRefObject; + /** Tracks in-flight timeline edits that patch the iframe DOM directly. File-change + * events for these paths are always suppressed since the preview is already up-to-date. */ + pendingTimelineEditPathRef?: React.MutableRefObject>; /** Called to reload the preview after undo/redo or external file changes. */ reloadPreview: () => void; } // ── Hook ── -export function useManifestPersistence({ +export function usePreviewPersistence({ projectId, showToast: _showToast, readOptionalProjectFile: _readOptionalProjectFile, @@ -44,7 +47,8 @@ export function useManifestPersistence({ activeCompPathRef: _activeCompPathRef, domEditSaveTimestampRef, reloadPreview, -}: UseManifestPersistenceParams) { + pendingTimelineEditPathRef, +}: UsePreviewPersistenceParams) { void _showToast; void _recordEdit; void _activeCompPathRef; @@ -162,8 +166,11 @@ export function useManifestPersistence({ const handler = (payload?: unknown) => { const changedPath = readStudioFileChangePath(payload); if (!changedPath) return; - const recentDomEditSave = Date.now() - domEditSaveTimestampRef.current < 1200; - // External file change — reload unless it's an echo of our own save. + const recentDomEditSave = Date.now() - domEditSaveTimestampRef.current < 4000; + if (pendingTimelineEditPathRef?.current.has(changedPath)) { + pendingTimelineEditPathRef.current.delete(changedPath); + return; + } if (!recentDomEditSave) { reloadPreview(); } diff --git a/packages/studio/src/hooks/useTimelineEditing.ts b/packages/studio/src/hooks/useTimelineEditing.ts index b2267378d..86e76f9f9 100644 --- a/packages/studio/src/hooks/useTimelineEditing.ts +++ b/packages/studio/src/hooks/useTimelineEditing.ts @@ -38,6 +38,8 @@ interface UseTimelineEditingOptions { recordEdit: (input: RecordEditInput) => Promise; domEditSaveTimestampRef: React.MutableRefObject; reloadPreview: () => void; + previewIframeRef: React.RefObject; + pendingTimelineEditPathRef: React.MutableRefObject>; uploadProjectFiles: (files: Iterable, dir?: string) => Promise; } @@ -53,6 +55,97 @@ function buildPatchTarget(element: { domId?: string; selector?: string; selector return null; } +// The runtime re-reads data-start/data-duration from the DOM on each sync tick +// (packages/core/src/runtime/init.ts:1324-1368), so attribute mutations here are +// picked up automatically on the next frame without a rebind call. +function patchIframeDomTiming( + iframe: HTMLIFrameElement | null, + element: TimelineElement, + attrs: Array<[string, string]>, +): void { + try { + const doc = iframe?.contentDocument; + if (!doc) return; + const el = element.domId + ? doc.getElementById(element.domId) + : element.selector + ? (doc.querySelectorAll(element.selector)[element.selectorIndex ?? 0] ?? null) + : null; + if (!el) return; + for (const [name, value] of attrs) el.setAttribute(name, value); + } catch { + // Cross-origin or mid-navigation — file save is enqueued; iframe patch is best-effort. + } +} + +function resolveResizePlaybackStart( + original: string, + target: PatchTarget, + element: TimelineElement, + updates: Pick, +): { attrName: string; value: number } | null { + if (updates.playbackStart != null) { + const attrName = + element.playbackStartAttr === "playback-start" ? "playback-start" : "media-start"; + return { attrName, value: updates.playbackStart }; + } + const trimDelta = updates.start - element.start; + if (trimDelta === 0) return null; + const raw = + readAttributeByTarget(original, target, "playback-start") ?? + readAttributeByTarget(original, target, "media-start"); + const current = raw != null ? parseFloat(raw) : undefined; + if (current == null || !Number.isFinite(current)) return null; + const attrName = + element.playbackStartAttr === "playback-start" ? "playback-start" : "media-start"; + return { + attrName, + value: Math.max(0, current + trimDelta * Math.max(element.playbackRate ?? 1, 0.1)), + }; +} + +type PatchTarget = NonNullable>; + +interface PersistTimelineEditInput { + projectId: string; + element: TimelineElement; + activeCompPath: string | null; + label: string; + buildPatches: (original: string, target: PatchTarget) => string; + writeProjectFile: (path: string, content: string) => Promise; + recordEdit: (input: RecordEditInput) => Promise; + domEditSaveTimestampRef: React.MutableRefObject; + pendingTimelineEditPathRef: React.MutableRefObject>; +} + +async function persistTimelineEdit(input: PersistTimelineEditInput): Promise { + const targetPath = input.element.sourceFile || input.activeCompPath || "index.html"; + const originalContent = await readFileContent(input.projectId, targetPath); + + const patchTarget = buildPatchTarget(input.element); + if (!patchTarget) { + throw new Error(`Timeline element ${input.element.id} is missing a patchable target`); + } + + const patchedContent = input.buildPatches(originalContent, patchTarget); + if (patchedContent === originalContent) { + throw new Error(`Unable to patch timeline element ${input.element.id} in ${targetPath}`); + } + + input.pendingTimelineEditPathRef.current.add(targetPath); + input.domEditSaveTimestampRef.current = Date.now(); + await saveProjectFilesWithHistory({ + projectId: input.projectId, + label: input.label, + kind: "timeline", + files: { [targetPath]: patchedContent }, + readFile: async () => originalContent, + writeFile: input.writeProjectFile, + recordEdit: input.recordEdit, + }); + input.domEditSaveTimestampRef.current = Date.now(); +} + async function readFileContent(projectId: string, targetPath: string): Promise { const response = await fetch( `/api/projects/${projectId}/files/${encodeURIComponent(targetPath)}`, @@ -78,127 +171,105 @@ export function useTimelineEditing({ recordEdit, domEditSaveTimestampRef, reloadPreview, + previewIframeRef, + pendingTimelineEditPathRef, uploadProjectFiles, }: UseTimelineEditingOptions) { const projectIdRef = useRef(projectId); projectIdRef.current = projectId; + const editQueueRef = useRef(Promise.resolve()); const lastBlockedTimelineToastAtRef = useRef(0); - const handleTimelineElementMove = useCallback( - async (element: TimelineElement, updates: Pick) => { + const enqueueEdit = useCallback( + ( + element: TimelineElement, + label: string, + buildPatches: PersistTimelineEditInput["buildPatches"], + ): Promise => { const pid = projectIdRef.current; - if (!pid) throw new Error("No active project"); - - const targetPath = element.sourceFile || activeCompPath || "index.html"; - const originalContent = await readFileContent(pid, targetPath); - - const patchTarget = buildPatchTarget(element); - if (!patchTarget) { - throw new Error(`Timeline element ${element.id} is missing a patchable target`); - } - - let patchedContent = applyPatchByTarget(originalContent, patchTarget, { - type: "attribute", - property: "start", - value: formatTimelineAttributeNumber(updates.start), - }); - patchedContent = applyPatchByTarget(patchedContent, patchTarget, { - type: "attribute", - property: "track-index", - value: String(updates.track), + if (!pid) return Promise.resolve(); + const queued = editQueueRef.current.then(() => + persistTimelineEdit({ + projectId: pid, + element, + activeCompPath, + label, + buildPatches, + writeProjectFile, + recordEdit, + domEditSaveTimestampRef, + pendingTimelineEditPathRef, + }), + ); + editQueueRef.current = queued.catch((error) => { + console.error(`[Timeline] Failed to persist: ${label}`, error); }); + return queued; + }, + [ + activeCompPath, + recordEdit, + writeProjectFile, + domEditSaveTimestampRef, + pendingTimelineEditPathRef, + ], + ); - if (patchedContent === originalContent) { - throw new Error(`Unable to patch timeline element ${element.id} in ${targetPath}`); - } - - domEditSaveTimestampRef.current = Date.now(); - await saveProjectFilesWithHistory({ - projectId: pid, - label: "Move timeline clip", - kind: "timeline", - files: { [targetPath]: patchedContent }, - readFile: async () => originalContent, - writeFile: writeProjectFile, - recordEdit, + const handleTimelineElementMove = useCallback( + (element: TimelineElement, updates: Pick) => { + patchIframeDomTiming(previewIframeRef.current, element, [ + ["data-start", formatTimelineAttributeNumber(updates.start)], + ["data-track-index", String(updates.track)], + ]); + return enqueueEdit(element, "Move timeline clip", (original, target) => { + let patched = applyPatchByTarget(original, target, { + type: "attribute", + property: "start", + value: formatTimelineAttributeNumber(updates.start), + }); + return applyPatchByTarget(patched, target, { + type: "attribute", + property: "track-index", + value: String(updates.track), + }); }); - - reloadPreview(); }, - [activeCompPath, recordEdit, writeProjectFile, domEditSaveTimestampRef, reloadPreview], + [previewIframeRef, enqueueEdit], ); const handleTimelineElementResize = useCallback( - async ( + ( element: TimelineElement, updates: Pick, ) => { - const pid = projectIdRef.current; - if (!pid) throw new Error("No active project"); - - const targetPath = element.sourceFile || activeCompPath || "index.html"; - const originalContent = await readFileContent(pid, targetPath); - - const patchTarget = buildPatchTarget(element); - if (!patchTarget) { - throw new Error(`Timeline element ${element.id} is missing a patchable target`); - } - - const playbackStartAttrName = - element.playbackStartAttr === "playback-start" ? "playback-start" : "media-start"; - const currentPlaybackStartValue = - readAttributeByTarget(originalContent, patchTarget, "playback-start") ?? - readAttributeByTarget(originalContent, patchTarget, "media-start"); - const currentPlaybackStart = - currentPlaybackStartValue != null ? parseFloat(currentPlaybackStartValue) : undefined; - const trimDelta = updates.start - element.start; - const fallbackPlaybackStart = - updates.playbackStart == null && - trimDelta !== 0 && - Number.isFinite(currentPlaybackStart) && - currentPlaybackStart != null - ? Math.max(0, currentPlaybackStart + trimDelta * Math.max(element.playbackRate ?? 1, 0.1)) - : undefined; - const nextPlaybackStart = updates.playbackStart ?? fallbackPlaybackStart; - - let patchedContent = originalContent; - patchedContent = applyPatchByTarget(patchedContent, patchTarget, { - type: "attribute", - property: "start", - value: formatTimelineAttributeNumber(updates.start), - }); - patchedContent = applyPatchByTarget(patchedContent, patchTarget, { - type: "attribute", - property: "duration", - value: formatTimelineAttributeNumber(updates.duration), - }); - if (nextPlaybackStart != null) { - patchedContent = applyPatchByTarget(patchedContent, patchTarget, { + patchIframeDomTiming(previewIframeRef.current, element, [ + ["data-start", formatTimelineAttributeNumber(updates.start)], + ["data-duration", formatTimelineAttributeNumber(updates.duration)], + ]); + return enqueueEdit(element, "Resize timeline clip", (original, target) => { + const pbs = resolveResizePlaybackStart(original, target, element, updates); + let patched = applyPatchByTarget(original, target, { type: "attribute", - property: playbackStartAttrName, - value: formatTimelineAttributeNumber(nextPlaybackStart), + property: "start", + value: formatTimelineAttributeNumber(updates.start), }); - } - - if (patchedContent === originalContent) { - throw new Error(`Unable to patch timeline element ${element.id} in ${targetPath}`); - } - - domEditSaveTimestampRef.current = Date.now(); - await saveProjectFilesWithHistory({ - projectId: pid, - label: "Resize timeline clip", - kind: "timeline", - files: { [targetPath]: patchedContent }, - readFile: async () => originalContent, - writeFile: writeProjectFile, - recordEdit, + patched = applyPatchByTarget(patched, target, { + type: "attribute", + property: "duration", + value: formatTimelineAttributeNumber(updates.duration), + }); + if (pbs) { + patched = applyPatchByTarget(patched, target, { + type: "attribute", + property: pbs.attrName, + value: formatTimelineAttributeNumber(pbs.value), + }); + } + return patched; }); - - reloadPreview(); }, - [activeCompPath, recordEdit, writeProjectFile, domEditSaveTimestampRef, reloadPreview], + [previewIframeRef, enqueueEdit], ); const handleTimelineElementDelete = useCallback( diff --git a/packages/studio/src/player/components/TimelineCanvas.tsx b/packages/studio/src/player/components/TimelineCanvas.tsx index ce9119331..d6d17a36e 100644 --- a/packages/studio/src/player/components/TimelineCanvas.tsx +++ b/packages/studio/src/player/components/TimelineCanvas.tsx @@ -252,6 +252,7 @@ export const TimelineCanvas = memo(function TimelineCanvas({ isHovered={hoveredClip === clipKey} isDragging={false} hasCustomContent={!!renderClipContent} + capabilities={capabilities} theme={theme} trackStyle={clipStyle} isComposition={isComposition} @@ -369,6 +370,7 @@ export const TimelineCanvas = memo(function TimelineCanvas({ isHovered={false} isDragging={true} hasCustomContent={!!renderClipContent} + capabilities={getTimelineEditCapabilities(activeDraggedElement)} theme={theme} trackStyle={getTrackStyle(activeDraggedElement.tag)} isComposition={!!activeDraggedElement.compositionSrc} diff --git a/packages/studio/src/player/components/TimelineClip.tsx b/packages/studio/src/player/components/TimelineClip.tsx index 0fbef33e0..de0c583b0 100644 --- a/packages/studio/src/player/components/TimelineClip.tsx +++ b/packages/studio/src/player/components/TimelineClip.tsx @@ -3,7 +3,7 @@ import type { TimelineTrackStyle } from "./timelineTheme"; import { memo, type ReactNode } from "react"; import type { TimelineElement } from "../store/playerStore"; import { defaultTimelineTheme, getClipHandleOpacity, type TimelineTheme } from "./timelineTheme"; -import { getTimelineEditCapabilities } from "./timelineEditing"; +import type { TimelineEditCapabilities } from "./timelineEditing"; interface TimelineClipProps { el: TimelineElement; @@ -13,6 +13,7 @@ interface TimelineClipProps { isHovered: boolean; isDragging?: boolean; hasCustomContent: boolean; + capabilities: TimelineEditCapabilities; theme?: TimelineTheme; trackStyle: TimelineTrackStyle; isComposition: boolean; @@ -33,6 +34,7 @@ export const TimelineClip = memo(function TimelineClip({ isHovered, isDragging = false, hasCustomContent, + capabilities, theme = defaultTimelineTheme, trackStyle, isComposition, @@ -47,6 +49,7 @@ export const TimelineClip = memo(function TimelineClip({ const leftPx = el.start * pps; const widthPx = Math.max(el.duration * pps, 4); const handleOpacity = getClipHandleOpacity({ isHovered, isSelected, isDragging }); + const borderColor = isSelected ? theme.clipBorderActive : isHovered @@ -59,7 +62,6 @@ export const TimelineClip = memo(function TimelineClip({ : isHovered ? theme.clipShadowHover : theme.clipShadow; - const capabilities = getTimelineEditCapabilities(el); const displayLabel = el.label || el.id || el.tag; const showHandles = handleOpacity > 0.01; diff --git a/packages/studio/src/player/components/timelineEditing.test.ts b/packages/studio/src/player/components/timelineEditing.test.ts index 68df1be9a..1e945c19b 100644 --- a/packages/studio/src/player/components/timelineEditing.test.ts +++ b/packages/studio/src/player/components/timelineEditing.test.ts @@ -4,7 +4,6 @@ import { buildPromptCopyText, buildTimelineElementAgentPrompt, buildTimelineAgentPrompt, - canOffsetTrimClipStart, getTimelineEditCapabilities, hasPatchableTimelineTarget, resolveBlockedTimelineEditIntent, @@ -158,42 +157,6 @@ describe("resolveTimelineMove", () => { }); }); -describe("canOffsetTrimClipStart", () => { - it("allows front trim for clips that carry playback offset metadata", () => { - expect( - canOffsetTrimClipStart({ - tag: "div", - playbackStartAttr: "media-start", - }), - ).toBe(true); - }); - - it("allows front trim for media clips with source duration metadata", () => { - expect( - canOffsetTrimClipStart({ - tag: "video", - sourceDuration: 12, - }), - ).toBe(true); - }); - - it("allows front trim for plain audio clips even before media-start exists", () => { - expect( - canOffsetTrimClipStart({ - tag: "audio", - }), - ).toBe(true); - }); - - it("blocks front trim for generic motion clips", () => { - expect( - canOffsetTrimClipStart({ - tag: "section", - }), - ).toBe(false); - }); -}); - describe("hasPatchableTimelineTarget", () => { it("returns true when the clip has a DOM id", () => { expect(hasPatchableTimelineTarget({ domId: "hero-card" })).toBe(true); @@ -224,7 +187,7 @@ describe("getTimelineEditCapabilities", () => { }); }); - it("allows moving generic motion clips while keeping trims blocked", () => { + it("allows full editing of generic motion clips with authored timing", () => { expect( getTimelineEditCapabilities({ tag: "section", @@ -233,8 +196,8 @@ describe("getTimelineEditCapabilities", () => { }), ).toEqual({ canMove: true, - canTrimStart: false, - canTrimEnd: false, + canTrimStart: true, + canTrimEnd: true, }); }); @@ -285,7 +248,7 @@ describe("getTimelineEditCapabilities", () => { }); }); - it("allows move and end trim for patchable composition hosts", () => { + it("allows full editing for patchable composition hosts", () => { expect( getTimelineEditCapabilities({ tag: "div", @@ -295,7 +258,22 @@ describe("getTimelineEditCapabilities", () => { }), ).toEqual({ canMove: true, - canTrimStart: false, + canTrimStart: true, + canTrimEnd: true, + }); + }); + + it("allows full editing of explicitly authored generic elements", () => { + expect( + getTimelineEditCapabilities({ + tag: "div", + duration: 4, + selector: "#hero-card", + timingSource: "authored", + }), + ).toEqual({ + canMove: true, + canTrimStart: true, canTrimEnd: true, }); }); @@ -576,6 +554,40 @@ describe("resolveTimelineResize", () => { ), ).toEqual({ start: 0.8, duration: 3.2, playbackStart: 0 }); }); + + it("trims generic element start without media offset", () => { + expect( + resolveTimelineResize( + { + start: 2, + duration: 4, + originClientX: 100, + pixelsPerSecond: 100, + minStart: 0, + maxEnd: 10, + }, + "start", + 200, + ), + ).toEqual({ start: 3, duration: 3, playbackStart: undefined }); + }); + + it("extends generic element start leftward to time zero", () => { + expect( + resolveTimelineResize( + { + start: 1, + duration: 3, + originClientX: 100, + pixelsPerSecond: 100, + minStart: 0, + maxEnd: 10, + }, + "start", + -200, + ), + ).toEqual({ start: 0, duration: 4, playbackStart: undefined }); + }); }); describe("buildPromptCopyText", () => { diff --git a/packages/studio/src/player/components/timelineEditing.ts b/packages/studio/src/player/components/timelineEditing.ts index d05d44a90..f837cb40b 100644 --- a/packages/studio/src/player/components/timelineEditing.ts +++ b/packages/studio/src/player/components/timelineEditing.ts @@ -201,18 +201,6 @@ export function hasPatchableTimelineTarget(input: { domId?: string; selector?: s return Boolean(input.domId || input.selector); } -export function canOffsetTrimClipStart(input: { - tag: string; - playbackStart?: number; - playbackStartAttr?: "media-start" | "playback-start"; - sourceDuration?: number; -}): boolean { - if (input.playbackStartAttr != null) return true; - if (input.playbackStart != null) return true; - const normalizedTag = input.tag.toLowerCase(); - return ["video", "audio"].includes(normalizedTag); -} - export function getTimelineEditCapabilities(input: { tag: string; duration: number; @@ -237,8 +225,8 @@ export function getTimelineEditCapabilities(input: { const hasDeterministicWindow = isDeterministicTimelineWindow(input); return { canMove: canPatch && (hasDeterministicWindow || hasFiniteDuration), - canTrimEnd: canPatch && hasFiniteDuration && hasDeterministicWindow, - canTrimStart: canPatch && hasFiniteDuration && canOffsetTrimClipStart(input), + canTrimEnd: canPatch && hasFiniteDuration, + canTrimStart: canPatch && hasFiniteDuration, }; } diff --git a/packages/studio/src/player/hooks/useTimelinePlayer.ts b/packages/studio/src/player/hooks/useTimelinePlayer.ts index 3c9db97d6..f503e8299 100644 --- a/packages/studio/src/player/hooks/useTimelinePlayer.ts +++ b/packages/studio/src/player/hooks/useTimelinePlayer.ts @@ -7,7 +7,7 @@ import { useTimelineSyncCallbacks } from "./useTimelineSyncCallbacks"; // Re-export public API consumed by tests and external modules. // All of these were previously defined in this file; they now live in focused // sub-modules but are re-exported here so existing import sites don't change. -export type { PlaybackAdapter, ClipManifestClip } from "../lib/playbackTypes"; +export type { ClipManifestClip } from "../lib/playbackTypes"; export { createStaticSeekPlaybackAdapter } from "../lib/playbackAdapter"; export { getTimelineElementSelector, @@ -42,6 +42,7 @@ import { setPreviewPlaybackRate, shouldMutePreviewAudio, } from "../lib/timelineIframeHelpers"; +import { probeMediaUrl, getCachedProbe } from "../lib/mediaProbe"; // --------------------------------------------------------------------------- // Hook @@ -106,6 +107,32 @@ export function useTimelinePlayer() { if (!state.timelineReady) { setTimelineReady(true); } + + // Asynchronously enrich media elements missing sourceDuration via mediabunny. + // The probe reads file headers only — no full decode — so this is cheap. + const needsProbe = mergedElements.filter( + (el) => + el.src && + el.sourceDuration == null && + ["video", "audio"].includes(el.tag.toLowerCase()) && + !getCachedProbe(el.src), + ); + if (needsProbe.length > 0) { + void Promise.allSettled( + needsProbe.map(async (el) => { + const result = await probeMediaUrl(el.src!); + if (!result) return; + const key = el.key ?? el.id; + usePlayerStore.setState((state) => { + const idx = state.elements.findIndex((e) => (e.key ?? e.id) === key); + if (idx === -1 || state.elements[idx].sourceDuration != null) return {}; + const patched = state.elements.slice(); + patched[idx] = { ...state.elements[idx], sourceDuration: result.duration }; + return { elements: patched }; + }); + }), + ); + } }, [setElements, setTimelineReady, setDuration], ); diff --git a/packages/studio/src/player/lib/mediaProbe.ts b/packages/studio/src/player/lib/mediaProbe.ts new file mode 100644 index 000000000..2ea95ae5c --- /dev/null +++ b/packages/studio/src/player/lib/mediaProbe.ts @@ -0,0 +1,68 @@ +import { Input, UrlSource, ALL_FORMATS } from "mediabunny"; + +export interface MediaProbeResult { + duration: number; + width?: number; + height?: number; + hasVideo: boolean; + hasAudio: boolean; +} + +const cache = new Map(); +const inflight = new Map>(); + +function normalizeUrl(url: string): string { + try { + return new URL(url, window.location.href).href; + } catch { + return url; + } +} + +async function probeOne(url: string): Promise { + const input = new Input({ + source: new UrlSource(url), + formats: ALL_FORMATS, + }); + try { + const duration = await input.getDurationFromMetadata(); + if (duration == null || !Number.isFinite(duration) || duration <= 0) return null; + + const videoTrack = await input.getPrimaryVideoTrack(); + const audioTracks = await input.getAudioTracks(); + + const result: MediaProbeResult = { + duration, + width: videoTrack?.displayWidth, + height: videoTrack?.displayHeight, + hasVideo: videoTrack != null, + hasAudio: audioTracks.length > 0, + }; + return result; + } catch { + return null; + } finally { + input.dispose(); + } +} + +export function getCachedProbe(url: string): MediaProbeResult | undefined { + return cache.get(normalizeUrl(url)); +} + +export async function probeMediaUrl(url: string): Promise { + const key = normalizeUrl(url); + const cached = cache.get(key); + if (cached) return cached; + + let pending = inflight.get(key); + if (pending) return pending; + + pending = probeOne(key).then((result) => { + inflight.delete(key); + if (result) cache.set(key, result); + return result; + }); + inflight.set(key, pending); + return pending; +}