diff --git a/electron/ipc/cursor/telemetry.ts b/electron/ipc/cursor/telemetry.ts index 51b901cd7..aa6f47848 100644 --- a/electron/ipc/cursor/telemetry.ts +++ b/electron/ipc/cursor/telemetry.ts @@ -26,7 +26,7 @@ export function clamp(value: number, min: number, max: number) { export function stopCursorCapture() { if (cursorCaptureInterval) { - clearInterval(cursorCaptureInterval); + clearTimeout(cursorCaptureInterval); setCursorCaptureInterval(null); } } @@ -155,13 +155,33 @@ export function snapshotCursorTelemetryForPersistence() { export function startCursorSampling() { stopCursorCapture(); - setCursorCaptureInterval( - setInterval(() => { - if (isCursorCaptureActive) { - sampleCursorPoint(); - } - }, CURSOR_SAMPLE_INTERVAL_MS), - ); + + // Use recursive setTimeout with drift compensation instead of setInterval. + // Under CPU load setInterval bunches or skips callbacks, creating large gaps + // in telemetry data. This approach measures wall-clock drift each tick and + // adjusts the next delay so samples stay close to the target interval. + let nextExpectedMs = Date.now() + CURSOR_SAMPLE_INTERVAL_MS; + + const tick = () => { + if (isCursorCaptureActive) { + sampleCursorPoint(); + } + + const now = Date.now(); + const drift = now - nextExpectedMs; + nextExpectedMs += CURSOR_SAMPLE_INTERVAL_MS; + + // If we fell behind by more than one full interval, reset the baseline + // so we don't try to "catch up" with a burst of rapid samples. + if (drift > CURSOR_SAMPLE_INTERVAL_MS) { + nextExpectedMs = now + CURSOR_SAMPLE_INTERVAL_MS; + } + + const delay = Math.max(1, nextExpectedMs - now); + setCursorCaptureInterval(setTimeout(tick, delay)); + }; + + setCursorCaptureInterval(setTimeout(tick, CURSOR_SAMPLE_INTERVAL_MS)); } // Re-export for consumers that use it from this module diff --git a/electron/ipc/ffmpeg/filters.ts b/electron/ipc/ffmpeg/filters.ts index dccbd8b40..a10212ba6 100644 --- a/electron/ipc/ffmpeg/filters.ts +++ b/electron/ipc/ffmpeg/filters.ts @@ -40,7 +40,7 @@ export function getAudioSyncAdjustment( const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000); const absDeltaMs = Math.abs(durationDeltaMs); - if (absDeltaMs <= 50) { + if (absDeltaMs <= 20) { return { mode: "none", delayMs: 0, tempoRatio: 1, durationDeltaMs }; } diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index d340b8e1d..0d70c5bed 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -194,53 +194,35 @@ export async function muxNativeMacRecordingWithAudio( tempoRatio: 1, durationDeltaMs: 0, }; - const needsFilter = systemAdjustment.mode !== "none" || micAdjustment.mode !== "none"; + // Always route through the filter graph so that aresample=async=1 is + // applied to every audio stream. This corrects progressive clock drift + // between the video and audio tracks that a simple duration comparison + // cannot detect (e.g. audio gradually falling behind under CPU load). let args: string[]; if (availableAudioInputs.length === 2) { - if (needsFilter) { - const filterParts: string[] = []; - appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment); - appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment); - filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]"); - args = [ - "-y", - ...inputs, - "-filter_complex", - filterParts.join(";"), - "-map", - "0:v:0", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ]; - } else { - args = [ - "-y", - ...inputs, - "-filter_complex", - "[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[aout]", - "-map", - "0:v:0", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ]; - } + const filterParts: string[] = []; + appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment); + appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment); + filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]"); + args = [ + "-y", + ...inputs, + "-filter_complex", + filterParts.join(";"), + "-map", + "0:v:0", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "192k", + "-shortest", + mixedOutputPath, + ]; } else { const singleAdjustment = audioAdjustments.get(availableAudioInputs[0]) ?? { mode: "none", @@ -248,45 +230,26 @@ export async function muxNativeMacRecordingWithAudio( tempoRatio: 1, durationDeltaMs: 0, }; - if (singleAdjustment.mode !== "none") { - const filterParts: string[] = []; - appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment); - args = [ - "-y", - ...inputs, - "-filter_complex", - filterParts.join(";"), - "-map", - "0:v:0", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ]; - } else { - args = [ - "-y", - ...inputs, - "-map", - "0:v:0", - "-map", - "1:a:0", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ]; - } + const filterParts: string[] = []; + appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment); + args = [ + "-y", + ...inputs, + "-filter_complex", + filterParts.join(";"), + "-map", + "0:v:0", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "192k", + "-shortest", + mixedOutputPath, + ]; } console.log("[mux] Running ffmpeg:", ffmpegPath, args.join(" ")); diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts index 312f31e16..80af66820 100644 --- a/electron/ipc/recording/windows.ts +++ b/electron/ipc/recording/windows.ts @@ -278,58 +278,38 @@ export async function muxNativeWindowsVideoWithAudio( durationDeltaMs: 0, }; - if (pauseFilter || singleAdjustment.mode !== "none") { - const filterParts: string[] = []; - if (pauseFilter) { - filterParts.push(pauseFilter); - } - const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]"; - appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment); - - await execFileAsync( - ffmpegPath, - [ - "-y", - ...inputs, - "-filter_complex", - filterParts.join(";"), - "-map", - "0:v:0", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ], - { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, - ); - } else { - await execFileAsync( - ffmpegPath, - [ - "-y", - ...inputs, - "-map", - "0:v:0", - "-map", - "1:a:0", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ], - { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, - ); + // Always route through the filter graph so that aresample=async=1 is + // applied. This corrects progressive clock drift between video and + // audio tracks that a simple duration comparison cannot detect. + const filterParts: string[] = []; + if (pauseFilter) { + filterParts.push(pauseFilter); } + const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]"; + appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment); + + await execFileAsync( + ffmpegPath, + [ + "-y", + ...inputs, + "-filter_complex", + filterParts.join(";"), + "-map", + "0:v:0", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "192k", + "-shortest", + mixedOutputPath, + ], + { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + ); } await moveFileWithOverwrite(mixedOutputPath, videoPath); diff --git a/electron/ipc/register/project.ts b/electron/ipc/register/project.ts index 1831247bc..32846a760 100644 --- a/electron/ipc/register/project.ts +++ b/electron/ipc/register/project.ts @@ -379,17 +379,23 @@ export function registerProjectHandlers() { } }); - ipcMain.handle('get-local-media-url', (_, filePath: string) => { + ipcMain.handle('get-local-media-url', async (_, filePath: string) => { const baseUrl = getMediaServerBaseUrl(); if (!baseUrl || !filePath) { return { success: false as const }; } - const resolved = path.resolve(filePath); - if (!approvedLocalReadPaths.has(resolved)) { + const normalized = path.resolve(filePath); + let resolved: string; + try { + resolved = await fs.realpath(normalized); + } catch { + return { success: false as const }; + } + if (!approvedLocalReadPaths.has(resolved) && !approvedLocalReadPaths.has(normalized)) { console.warn(`[get-local-media-url] Blocked unapproved path: ${resolved}`); return { success: false as const }; } - return { success: true as const, url: buildMediaUrl(baseUrl, filePath) }; + return { success: true as const, url: buildMediaUrl(baseUrl, resolved) }; }); } diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index c15510588..78184758d 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { promisify } from "node:util"; import { app, BrowserWindow, desktopCapturer, dialog, ipcMain, shell, systemPreferences } from "electron"; import { showCursor } from "../../cursorHider"; -import { ALLOW_RECORDLY_WINDOW_CAPTURE, CURSOR_SAMPLE_INTERVAL_MS } from "../constants"; +import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants"; import type { SelectedSource, NativeMacRecordingOptions, PauseSegment, CursorTelemetryPoint } from "../types"; import { selectedSource, @@ -53,7 +53,6 @@ import { setCachedSystemCursorAssets, cachedSystemCursorAssetsSourceMtimeMs, setCachedSystemCursorAssetsSourceMtimeMs, - setCursorCaptureInterval, setCursorCaptureStartTimeMs, setActiveCursorSamples, setPendingCursorSamples, @@ -113,6 +112,7 @@ import { clamp, stopCursorCapture, sampleCursorPoint, + startCursorSampling, snapshotCursorTelemetryForPersistence, } from "../cursor/telemetry"; import { @@ -1083,7 +1083,7 @@ export function registerRecordingHandlers( setLinuxCursorScreenPoint(null) setLastLeftClick(null) sampleCursorPoint() - setCursorCaptureInterval(setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS)) + startCursorSampling() void startInteractionCapture() } else { setIsCursorCaptureActive(false) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index d49326093..3edc04273 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -781,7 +781,7 @@ export default function VideoEditor() { padding, cropRegion, webcam, - webcamUrl: webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null, + webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), videoWidth: previewVideo.videoWidth, videoHeight: previewVideo.videoHeight, annotationRegions, @@ -1807,7 +1807,6 @@ export default function VideoEditor() { setResolvedWebcamVideoUrl(null); return; } - setResolvedWebcamVideoUrl(null); void resolveVideoUrl(webcam.sourcePath).then((url) => { if (!cancelled) setResolvedWebcamVideoUrl(url); }); @@ -3540,7 +3539,7 @@ export default function VideoEditor() { videoPadding: padding, cropRegion, webcam, - webcamUrl: webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null, + webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), annotationRegions, autoCaptions, autoCaptionSettings, @@ -3709,7 +3708,7 @@ export default function VideoEditor() { padding, cropRegion, webcam, - webcamUrl: webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null, + webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), annotationRegions, autoCaptions, autoCaptionSettings, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 5070bd114..926845798 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -18,7 +18,7 @@ const HIGH_FRAME_RATE_BOOST = 1.7; const DEFAULT_WIDTH = 1920; const DEFAULT_HEIGHT = 1080; const CODEC_ALIGNMENT = 2; -const RECORDER_TIMESLICE_MS = 1000; +const RECORDER_TIMESLICE_MS = 250; const BITS_PER_MEGABIT = 1_000_000; const MIN_FRAME_RATE = 30; const CHROME_MEDIA_SOURCE = "desktop"; @@ -968,7 +968,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackChunks.current.push(event.data); } }; - recorder.start(1000); + recorder.start(RECORDER_TIMESLICE_MS); micFallbackRecorder.current = recorder; } catch (micError) { console.warn("Browser microphone fallback failed:", micError); @@ -1090,7 +1090,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const micAudioTrack = microphoneStream.current?.getAudioTracks()[0]; if (systemAudioTrack && micAudioTrack) { - const context = new AudioContext(); + const context = new AudioContext({ sampleRate: 48000 }); mixingContext.current = context; const systemSource = context.createMediaStreamSource( new MediaStream([systemAudioTrack]), diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 16d919f33..d054f9c04 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -18,6 +18,12 @@ const DECODE_BACKPRESSURE_LIMIT = 20; const ENCODE_BACKPRESSURE_LIMIT = 20; const MIN_SPEED_REGION_DELTA_MS = 0.0001; const MP4_AUDIO_CODEC = "mp4a.40.2"; +const SYNC_SEEK_THRESHOLD_SEC = 0.15; +const SYNC_PLAYBACK_RATE_OPTIONS = { + toleranceSeconds: 0.008, + correctionWindowSeconds: 0.5, + maxAdjustment: 0.12, +} as const; export async function isAacAudioEncodingSupported( sampleRate = 48_000, @@ -555,7 +561,7 @@ export class AudioProcessor { throw new Error("Export cancelled"); } - audioContext = new AudioContext(); + audioContext = new AudioContext({ sampleRate: 48000 }); const currentDestinationNode = audioContext.createMediaStreamDestination(); destinationNode = currentDestinationNode; @@ -721,7 +727,7 @@ export class AudioProcessor { continue; } - if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) { + if (Math.abs(audioEl.currentTime - targetTimeSec) > SYNC_SEEK_THRESHOLD_SEC) { audioEl.currentTime = targetTimeSec; } @@ -729,6 +735,7 @@ export class AudioProcessor { basePlaybackRate: playbackRate, currentTime: audioEl.currentTime, targetTime: targetTimeSec, + ...SYNC_PLAYBACK_RATE_OPTIONS, }); if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { audioEl.playbackRate = syncedPlaybackRate; @@ -748,7 +755,7 @@ export class AudioProcessor { if (isInRegion) { const audioOffset = (currentTimeMs - region.startMs) / 1000; - if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) { + if (Math.abs(audioEl.currentTime - audioOffset) > SYNC_SEEK_THRESHOLD_SEC) { audioEl.currentTime = audioOffset; } @@ -756,6 +763,7 @@ export class AudioProcessor { basePlaybackRate: playbackRate, currentTime: audioEl.currentTime, targetTime: audioOffset, + ...SYNC_PLAYBACK_RATE_OPTIONS, }); if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { audioEl.playbackRate = syncedPlaybackRate; diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index 40e60cc1d..26215922b 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -2336,21 +2336,53 @@ export class FrameRenderer { return null; } - let closest = telemetry[0]; - let minDist = Math.abs(telemetry[0].timeMs - timeMs); - for (let i = 1; i < telemetry.length; i++) { - const dist = Math.abs(telemetry[i].timeMs - timeMs); - if (dist < minDist) { - minDist = dist; - closest = telemetry[i]; - } - if (telemetry[i].timeMs > timeMs) { - break; + // Clamp to first/last sample when out of range + if (timeMs <= telemetry[0].timeMs) { + const s = telemetry[0]; + return mapCursorToCanvasNormalized( + { cx: s.cx, cy: s.cy, interactionType: s.interactionType }, + { + maskRect: this.layoutCache?.maskRect, + canvasWidth: this.config.width, + canvasHeight: this.config.height, + }, + ); + } + if (timeMs >= telemetry[telemetry.length - 1].timeMs) { + const s = telemetry[telemetry.length - 1]; + return mapCursorToCanvasNormalized( + { cx: s.cx, cy: s.cy, interactionType: s.interactionType }, + { + maskRect: this.layoutCache?.maskRect, + canvasWidth: this.config.width, + canvasHeight: this.config.height, + }, + ); + } + + // Binary search for surrounding samples + let lo = 0; + let hi = telemetry.length - 1; + while (lo < hi - 1) { + const mid = (lo + hi) >> 1; + if (telemetry[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid; } } + const a = telemetry[lo]; + const b = telemetry[hi]; + const span = b.timeMs - a.timeMs; + + // Linear interpolation between samples + const t = span > 0 ? (timeMs - a.timeMs) / span : 0; + const cx = a.cx + (b.cx - a.cx) * t; + const cy = a.cy + (b.cy - a.cy) * t; + return mapCursorToCanvasNormalized( - { cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType }, + { cx, cy, interactionType: a.interactionType }, { maskRect: this.layoutCache?.maskRect, canvasWidth: this.config.width, diff --git a/src/lib/extensions/extensionHost.ts b/src/lib/extensions/extensionHost.ts index 32b327df1..9361c22de 100644 --- a/src/lib/extensions/extensionHost.ts +++ b/src/lib/extensions/extensionHost.ts @@ -854,17 +854,31 @@ export class ExtensionHost { const t = host._cursorTelemetry; if (!t || t.length === 0) return null; - let closest = t[0]; - let minDist = Math.abs(t[0].timeMs - timeMs); - for (let i = 1; i < t.length; i++) { - const dist = Math.abs(t[i].timeMs - timeMs); - if (dist < minDist) { - minDist = dist; - closest = t[i]; + if (timeMs <= t[0].timeMs) return { ...t[0], timeMs }; + if (timeMs >= t[t.length - 1].timeMs) return { ...t[t.length - 1], timeMs }; + + let lo = 0; + let hi = t.length - 1; + while (lo < hi - 1) { + const mid = (lo + hi) >> 1; + if (t[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid; } - if (t[i].timeMs > timeMs) break; } - return { ...closest }; + + const a = t[lo]; + const b = t[hi]; + const span = b.timeMs - a.timeMs; + const frac = span > 0 ? (timeMs - a.timeMs) / span : 0; + + return { + ...a, + cx: a.cx + (b.cx - a.cx) * frac, + cy: a.cy + (b.cy - a.cy) * frac, + timeMs, + }; }, getSmoothedCursor() { diff --git a/src/lib/mediaTiming.test.ts b/src/lib/mediaTiming.test.ts index 1f74a2592..628858636 100644 --- a/src/lib/mediaTiming.test.ts +++ b/src/lib/mediaTiming.test.ts @@ -23,10 +23,11 @@ describe("clampMediaTimeToDuration", () => { describe("estimateCompanionAudioStartDelaySeconds", () => { it("returns the positive tail gap when companion audio is shorter", () => { expect(estimateCompanionAudioStartDelaySeconds(10, 9.6)).toBeCloseTo(0.4); + expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBeCloseTo(0.03); }); it("ignores tiny or negative differences", () => { - expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBe(0); + expect(estimateCompanionAudioStartDelaySeconds(10, 9.99)).toBe(0); expect(estimateCompanionAudioStartDelaySeconds(10, 10.5)).toBe(0); }); }); diff --git a/src/lib/mediaTiming.ts b/src/lib/mediaTiming.ts index 11b6a50e8..ed3c0ff4c 100644 --- a/src/lib/mediaTiming.ts +++ b/src/lib/mediaTiming.ts @@ -19,7 +19,7 @@ export function estimateCompanionAudioStartDelaySeconds( const safeAudioDuration = Math.max(0, audioDuration ?? 0); const estimatedDelaySeconds = safeTimelineDuration - safeAudioDuration; - return estimatedDelaySeconds > 0.05 ? estimatedDelaySeconds : 0; + return estimatedDelaySeconds > 0.025 ? estimatedDelaySeconds : 0; } export function getMediaSyncPlaybackRate({