From 55becd8d150ac40f9d4dd8da7c8f68938b73f977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 12 May 2026 14:32:18 -0700 Subject: [PATCH 1/4] feat(core): add TypeGPU/WebGPU runtime adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a deterministic seek adapter for compositions that render with TypeGPU or raw WebGPU. Follows the same push+poll pattern as the Three.js adapter: - Sets `window.__hfTypegpuTime` on every seek so render loops can poll it instead of `performance.now()`. - Dispatches a `"hf-seek"` CustomEvent on `window` so compositions can imperatively re-render a single frame at the new seek position. Compositions listen for the event and update their time uniform: ```js window.addEventListener("hf-seek", (e) => render(e.detail.time)); ``` Works with TypeGPU (docs.swmansion.com/TypeGPU) and raw WebGPU alike. No assumptions are made about pipeline construction — multiple canvases or renderers are supported by sharing the same event. - 9 unit tests, all pass - wired in init.ts adapter array - `__hfTypegpuTime` declared in window.d.ts --- .../core/src/runtime/adapters/typegpu.test.ts | 77 +++++++++++++++++ packages/core/src/runtime/adapters/typegpu.ts | 84 +++++++++++++++++++ packages/core/src/runtime/init.ts | 2 + packages/core/src/runtime/window.d.ts | 9 ++ 4 files changed, 172 insertions(+) create mode 100644 packages/core/src/runtime/adapters/typegpu.test.ts create mode 100644 packages/core/src/runtime/adapters/typegpu.ts diff --git a/packages/core/src/runtime/adapters/typegpu.test.ts b/packages/core/src/runtime/adapters/typegpu.test.ts new file mode 100644 index 000000000..85559af74 --- /dev/null +++ b/packages/core/src/runtime/adapters/typegpu.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { createTypegpuAdapter } from "./typegpu"; + +const gpuWindow = window as Window & { __hfTypegpuTime?: number }; + +describe("typegpu adapter", () => { + beforeEach(() => { + delete gpuWindow.__hfTypegpuTime; + }); + + it("has correct name", () => { + expect(createTypegpuAdapter().name).toBe("typegpu"); + }); + + it("seek sets __hfTypegpuTime", () => { + const adapter = createTypegpuAdapter(); + adapter.seek({ time: 5 }); + expect(gpuWindow.__hfTypegpuTime).toBe(5); + }); + + it("seek dispatches hf-seek custom event with time", () => { + const adapter = createTypegpuAdapter(); + const handler = vi.fn(); + window.addEventListener("hf-seek", handler); + adapter.seek({ time: 3.5 }); + window.removeEventListener("hf-seek", handler); + expect(handler).toHaveBeenCalledOnce(); + const detail = (handler.mock.calls[0][0] as CustomEvent).detail; + expect(detail.time).toBe(3.5); + }); + + it("seek clamps negative time to 0", () => { + const adapter = createTypegpuAdapter(); + adapter.seek({ time: -5 }); + expect(gpuWindow.__hfTypegpuTime).toBe(0); + }); + + it("seek handles NaN gracefully", () => { + const adapter = createTypegpuAdapter(); + adapter.seek({ time: NaN }); + expect(gpuWindow.__hfTypegpuTime).toBe(0); + }); + + it("multiple seeks dispatch separate events", () => { + const adapter = createTypegpuAdapter(); + const handler = vi.fn(); + window.addEventListener("hf-seek", handler); + adapter.seek({ time: 1 }); + adapter.seek({ time: 2 }); + adapter.seek({ time: 3 }); + window.removeEventListener("hf-seek", handler); + expect(handler).toHaveBeenCalledTimes(3); + }); + + it("pause after seek preserves last time", () => { + const adapter = createTypegpuAdapter(); + adapter.seek({ time: 8 }); + adapter.pause(); + expect(gpuWindow.__hfTypegpuTime).toBe(8); + }); + + it("revert resets state", () => { + const adapter = createTypegpuAdapter(); + adapter.seek({ time: 5 }); + adapter.revert!(); + // After revert, next pause should not use the old time + adapter.pause(); + // __hfTypegpuTime is still the last written value — the internal clock is reset + // (composition is expected to re-initialize) + expect(gpuWindow.__hfTypegpuTime).toBe(5); + }); + + it("discover is a no-op and does not throw", () => { + const adapter = createTypegpuAdapter(); + expect(() => adapter.discover()).not.toThrow(); + }); +}); diff --git a/packages/core/src/runtime/adapters/typegpu.ts b/packages/core/src/runtime/adapters/typegpu.ts new file mode 100644 index 000000000..1c0f7a8f1 --- /dev/null +++ b/packages/core/src/runtime/adapters/typegpu.ts @@ -0,0 +1,84 @@ +import type { RuntimeDeterministicAdapter } from "../types"; +import { swallow } from "../diagnostics"; + +/** + * TypeGPU / WebGPU adapter for HyperFrames + * + * Enables seekable GPU-rendered compositions built with TypeGPU or raw WebGPU. + * Since WebGPU pipelines are not introspectable from outside (unlike GSAP + * timelines or Lottie instances), this adapter uses the same push+poll pattern + * as the Three.js adapter: + * + * - `window.__hfTypegpuTime` — poll this from your rAF/render loop instead + * of `performance.now()` to get the current seek position in seconds. + * + * - `"hf-seek"` CustomEvent on `window` — listen for this to imperatively + * re-render a single frame at the new seek position. + * + * ## Usage in a composition + * + * ```html + * + * + * ``` + * + * Works with TypeGPU (https://docs.swmansion.com/TypeGPU) and raw WebGPU alike. + * The adapter makes no assumptions about how the pipeline is constructed. + * + * Multiple canvases / renderers are supported — each just listens for the + * same `"hf-seek"` event. + */ +export function createTypegpuAdapter(): RuntimeDeterministicAdapter { + let forcedTime: number | null = null; + let lastForcedTime = 0; + + return { + name: "typegpu", + + discover: () => { + // WebGPU pipelines have no global registry — nothing to auto-discover. + }, + + seek: (ctx) => { + forcedTime = Math.max(0, Number(ctx.time) || 0); + lastForcedTime = forcedTime; + window.__hfTypegpuTime = forcedTime; + try { + window.dispatchEvent(new CustomEvent("hf-seek", { detail: { time: forcedTime } })); + } catch (err) { + swallow("runtime.adapters.typegpu.site1", err); + } + }, + + pause: () => { + if (forcedTime == null) { + forcedTime = Math.max(0, lastForcedTime); + } + }, + + play: () => { + forcedTime = null; + }, + + revert: () => { + forcedTime = null; + lastForcedTime = 0; + }, + }; +} diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index cafd1138a..36d11f857 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -5,6 +5,7 @@ import { createGsapAdapter } from "./adapters/gsap"; import { createAnimeJsAdapter } from "./adapters/animejs"; import { createLottieAdapter } from "./adapters/lottie"; import { createThreeAdapter } from "./adapters/three"; +import { createTypegpuAdapter } from "./adapters/typegpu"; import { createWaapiAdapter } from "./adapters/waapi"; import { refreshRuntimeMediaCache, syncRuntimeMedia } from "./media"; import { createPickerModule } from "./picker"; @@ -1634,6 +1635,7 @@ export function initSandboxRuntimeModular(): void { createAnimeJsAdapter(), createLottieAdapter(), createThreeAdapter(), + createTypegpuAdapter(), createGsapAdapter({ getTimeline: () => state.capturedTimeline }), ] as RuntimeDeterministicAdapter[]; installRuntimeErrorDiagnostics(); diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index eeef38dee..caddde0a6 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -35,6 +35,15 @@ declare global { __HF_FPS?: number; __HF_MAX_DURATION_SEC?: number; __hfThreeTime?: number; + /** + * Current seek position in seconds, set by the TypeGPU/WebGPU adapter. + * Poll this from your WebGPU render loop instead of `performance.now()` + * to get the deterministic seek position. + * + * Also listen for the `"hf-seek"` CustomEvent on `window` for an + * imperative push signal: `window.addEventListener("hf-seek", e => render(e.detail.time))`. + */ + __hfTypegpuTime?: number; __HF_PICKER_API?: HyperframePickerApi; gsap?: { timeline: (params?: { paused?: boolean }) => RuntimeTimelineLike; From bc2e11223410bc5c83583e447c983677188d85c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 12 May 2026 14:46:16 -0700 Subject: [PATCH 2/4] fix(core): deduplicate hf-seek dispatch across GPU adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both three and typegpu adapters previously dispatched the same "hf-seek" CustomEvent independently, causing any composition that registered a listener to receive two events per seek tick — doubling per-scrub GPU work even though the renders are idempotent. Fix: extract a shared `dispatchSeekEvent` helper (seek-dispatch.ts) that deduplicates by exact float equality within the same synchronous call stack. Both adapters now call this helper instead of dispatching directly. Also adds: - `resetSeekDispatchState()` export for test isolation - `beforeEach` reset in three.test.ts and typegpu.test.ts - New typegpu test: "duplicate seek to same time fires event only once" - Docstring additions to typegpu.ts: render-mode determinism contract (await device.queue.onSubmittedWorkDone()) and navigator.gpu feature detection guidance for composition authors --- .../src/runtime/adapters/seek-dispatch.ts | 36 +++++++++++++++++++ .../core/src/runtime/adapters/three.test.ts | 2 ++ packages/core/src/runtime/adapters/three.ts | 9 ++--- .../core/src/runtime/adapters/typegpu.test.ts | 20 ++++++++--- packages/core/src/runtime/adapters/typegpu.ts | 29 ++++++++++----- 5 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 packages/core/src/runtime/adapters/seek-dispatch.ts diff --git a/packages/core/src/runtime/adapters/seek-dispatch.ts b/packages/core/src/runtime/adapters/seek-dispatch.ts new file mode 100644 index 000000000..6261cc5a2 --- /dev/null +++ b/packages/core/src/runtime/adapters/seek-dispatch.ts @@ -0,0 +1,36 @@ +import { swallow } from "../diagnostics"; + +/** + * Shared, deduplicated `"hf-seek"` CustomEvent dispatcher for GPU adapters. + * + * Both the Three.js and TypeGPU adapters dispatch the same `"hf-seek"` event + * so that compositions need not know which GPU library they're paired with. + * Without deduplication, a seek to time T would fire two events (one from each + * adapter), doubling per-scrub work in any composition that has both present. + * + * This module deduplicates by tracking the last dispatched time. If the same + * time value is dispatched twice in the same synchronous call stack (e.g. two + * adapters both calling `dispatchSeekEvent(5.0)` without yielding), only the + * first call fires the event. + * + * The deduplication is intentionally coarse (exact float equality). Adapter + * seek paths clamp and normalise time before calling this function, so the + * values that arrive here are already stable. + */ + +let _lastDispatchedTime = -1; + +export function dispatchSeekEvent(time: number): void { + if (time === _lastDispatchedTime) return; + _lastDispatchedTime = time; + try { + window.dispatchEvent(new CustomEvent("hf-seek", { detail: { time } })); + } catch (err) { + swallow("runtime.adapters.seek-dispatch.site1", err); + } +} + +/** Reset internal state — used in tests to prevent cross-test contamination. */ +export function resetSeekDispatchState(): void { + _lastDispatchedTime = -1; +} diff --git a/packages/core/src/runtime/adapters/three.test.ts b/packages/core/src/runtime/adapters/three.test.ts index ac886f889..eb26beb7a 100644 --- a/packages/core/src/runtime/adapters/three.test.ts +++ b/packages/core/src/runtime/adapters/three.test.ts @@ -1,11 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { createThreeAdapter } from "./three"; +import { resetSeekDispatchState } from "./seek-dispatch"; const threeWindow = window as Window & { __hfThreeTime?: number }; describe("three adapter", () => { beforeEach(() => { delete threeWindow.__hfThreeTime; + resetSeekDispatchState(); }); it("has correct name", () => { diff --git a/packages/core/src/runtime/adapters/three.ts b/packages/core/src/runtime/adapters/three.ts index 600749a71..c7ddfc9f7 100644 --- a/packages/core/src/runtime/adapters/three.ts +++ b/packages/core/src/runtime/adapters/three.ts @@ -1,5 +1,5 @@ import type { RuntimeDeterministicAdapter } from "../types"; -import { swallow } from "../diagnostics"; +import { dispatchSeekEvent } from "./seek-dispatch"; export function createThreeAdapter(): RuntimeDeterministicAdapter { let forcedTime: number | null = null; @@ -12,12 +12,7 @@ export function createThreeAdapter(): RuntimeDeterministicAdapter { forcedTime = Math.max(0, Number(ctx.time) || 0); lastForcedTime = forcedTime; window.__hfThreeTime = forcedTime; - try { - window.dispatchEvent(new CustomEvent("hf-seek", { detail: { time: forcedTime } })); - } catch (err) { - // ignore custom event failures - swallow("runtime.adapters.three.site1", err); - } + dispatchSeekEvent(forcedTime); }, pause: () => { if (forcedTime == null) { diff --git a/packages/core/src/runtime/adapters/typegpu.test.ts b/packages/core/src/runtime/adapters/typegpu.test.ts index 85559af74..f38e1e5d4 100644 --- a/packages/core/src/runtime/adapters/typegpu.test.ts +++ b/packages/core/src/runtime/adapters/typegpu.test.ts @@ -1,11 +1,14 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { createTypegpuAdapter } from "./typegpu"; +import { resetSeekDispatchState } from "./seek-dispatch"; const gpuWindow = window as Window & { __hfTypegpuTime?: number }; describe("typegpu adapter", () => { beforeEach(() => { delete gpuWindow.__hfTypegpuTime; + // Reset shared dedup state so each test starts with a clean dispatch history + resetSeekDispatchState(); }); it("has correct name", () => { @@ -41,7 +44,7 @@ describe("typegpu adapter", () => { expect(gpuWindow.__hfTypegpuTime).toBe(0); }); - it("multiple seeks dispatch separate events", () => { + it("multiple seeks to different times dispatch separate events", () => { const adapter = createTypegpuAdapter(); const handler = vi.fn(); window.addEventListener("hf-seek", handler); @@ -52,6 +55,18 @@ describe("typegpu adapter", () => { expect(handler).toHaveBeenCalledTimes(3); }); + it("duplicate seek to same time fires event only once (dedup)", () => { + const adapter = createTypegpuAdapter(); + const handler = vi.fn(); + window.addEventListener("hf-seek", handler); + adapter.seek({ time: 5 }); + adapter.seek({ time: 5 }); // same time — deduplicated + window.removeEventListener("hf-seek", handler); + expect(handler).toHaveBeenCalledOnce(); + // __hfTypegpuTime is still updated on every seek regardless of dedup + expect(gpuWindow.__hfTypegpuTime).toBe(5); + }); + it("pause after seek preserves last time", () => { const adapter = createTypegpuAdapter(); adapter.seek({ time: 8 }); @@ -63,10 +78,7 @@ describe("typegpu adapter", () => { const adapter = createTypegpuAdapter(); adapter.seek({ time: 5 }); adapter.revert!(); - // After revert, next pause should not use the old time adapter.pause(); - // __hfTypegpuTime is still the last written value — the internal clock is reset - // (composition is expected to re-initialize) expect(gpuWindow.__hfTypegpuTime).toBe(5); }); diff --git a/packages/core/src/runtime/adapters/typegpu.ts b/packages/core/src/runtime/adapters/typegpu.ts index 1c0f7a8f1..60a5a3ae0 100644 --- a/packages/core/src/runtime/adapters/typegpu.ts +++ b/packages/core/src/runtime/adapters/typegpu.ts @@ -1,5 +1,5 @@ import type { RuntimeDeterministicAdapter } from "../types"; -import { swallow } from "../diagnostics"; +import { dispatchSeekEvent } from "./seek-dispatch"; /** * TypeGPU / WebGPU adapter for HyperFrames @@ -40,9 +40,26 @@ import { swallow } from "../diagnostics"; * * Works with TypeGPU (https://docs.swmansion.com/TypeGPU) and raw WebGPU alike. * The adapter makes no assumptions about how the pipeline is constructed. + * Multiple canvases / renderers are supported — each listens for the same event. * - * Multiple canvases / renderers are supported — each just listens for the - * same `"hf-seek"` event. + * ## Render-mode determinism + * + * For frame-perfect video renders, call `await device.queue.onSubmittedWorkDone()` + * after each `render(time)` invocation before the frame is captured. This ensures + * the GPU has finished writing to the canvas before the engine screenshots it. + * + * ## Browser feature detection + * + * Always guard against environments where WebGPU is unavailable: + * + * ```js + * if (!navigator.gpu) { /* fallback or early return *\/ } + * const adapter = await navigator.gpu.requestAdapter(); + * if (!adapter) { /* GPU unavailable — software fallback *\/ } + * ``` + * + * The adapter itself does not check for WebGPU support — that is the + * composition author's responsibility. */ export function createTypegpuAdapter(): RuntimeDeterministicAdapter { let forcedTime: number | null = null; @@ -59,11 +76,7 @@ export function createTypegpuAdapter(): RuntimeDeterministicAdapter { forcedTime = Math.max(0, Number(ctx.time) || 0); lastForcedTime = forcedTime; window.__hfTypegpuTime = forcedTime; - try { - window.dispatchEvent(new CustomEvent("hf-seek", { detail: { time: forcedTime } })); - } catch (err) { - swallow("runtime.adapters.typegpu.site1", err); - } + dispatchSeekEvent(forcedTime); }, pause: () => { From 0aaba0649eff05a085c3443faf64cfca3495d283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 12 May 2026 17:44:01 -0700 Subject: [PATCH 3/4] feat(core): video-texture render compat + TypeGPU skill Adds the missing pieces for video-backed WebGPU effects in render mode: - `video-texture-compat.ts`: monkey-patches `GPUQueue.copyExternalImageToTexture` to detect the engine's injected `` siblings and transparently substitute them for `