From fcdc842371d2c3c2580653817f7a64d84b9d797a Mon Sep 17 00:00:00 2001 From: wiiiii123 Date: Tue, 21 Apr 2026 00:08:25 +0700 Subject: [PATCH 1/5] fix(export): fast-path simple edited audio tracks --- electron/electron-env.d.ts | 6 + electron/ipc/export/native-video.ts | 14 +- electron/ipc/nativeVideoExport.test.ts | 43 ++++++ electron/ipc/nativeVideoExport.ts | 58 ++++++++ electron/preload.ts | 6 + src/lib/exporter/editedTrackStrategy.test.ts | 121 ++++++++++++++++ src/lib/exporter/editedTrackStrategy.ts | 141 +++++++++++++++++++ src/lib/exporter/modernVideoExporter.ts | 108 +++++++++++++- src/lib/exporter/streamingDecoder.ts | 5 + src/lib/exporter/videoExporter.ts | 106 +++++++++++++- 10 files changed, 595 insertions(+), 13 deletions(-) create mode 100644 electron/ipc/nativeVideoExport.test.ts create mode 100644 src/lib/exporter/editedTrackStrategy.test.ts create mode 100644 src/lib/exporter/editedTrackStrategy.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index f37eae2b1..5012dbe5c 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -194,7 +194,10 @@ interface Window { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, @@ -213,7 +216,10 @@ interface Window { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, diff --git a/electron/ipc/export/native-video.ts b/electron/ipc/export/native-video.ts index 77552cbb6..a92053698 100644 --- a/electron/ipc/export/native-video.ts +++ b/electron/ipc/export/native-video.ts @@ -14,6 +14,7 @@ import type { NativeVideoExportFinishOptions, } from "../nativeVideoExport"; import { + buildEditedTrackSourceAudioFilter, buildNativeVideoExportArgs, buildTrimmedSourceAudioFilter, getEditedAudioExtension, @@ -387,8 +388,10 @@ export async function muxNativeVideoExportAudio( const metrics: NativeVideoAudioMuxMetrics = {}; const tempArtifacts: string[] = []; let audioInputPath = options.audioSourcePath ?? null; + const useEditedTrackFiltergraph = + audioMode === "edited-track" && options.editedTrackStrategy === "filtergraph-fast-path"; - if (audioMode === "edited-track") { + if (audioMode === "edited-track" && !useEditedTrackFiltergraph) { if (!options.editedAudioData) { throw new Error("Edited audio data is missing for native export"); } @@ -435,6 +438,15 @@ export async function muxNativeVideoExportAudio( } else { args.push("-map", "0:v:0", "-map", "1:a:0"); } + } else if (useEditedTrackFiltergraph) { + const filter = buildEditedTrackSourceAudioFilter( + options.editedTrackSegments ?? [], + options.audioSourceSampleRate ?? 0, + ); + if (!filter) { + throw new Error("Edited-track filtergraph inputs are incomplete for native export"); + } + args.push("-filter_complex", filter, "-map", "0:v:0", "-map", "[aout]"); } else { args.push("-map", "0:v:0", "-map", "1:a:0"); } diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts new file mode 100644 index 000000000..1d0630d98 --- /dev/null +++ b/electron/ipc/nativeVideoExport.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { + buildEditedTrackSourceAudioFilter, + buildTrimmedSourceAudioFilter, +} from "./nativeVideoExport"; + +describe("buildTrimmedSourceAudioFilter", () => { + it("concatenates trimmed source segments into a single output label", () => { + expect( + buildTrimmedSourceAudioFilter([ + { startMs: 0, endMs: 2_000 }, + { startMs: 4_000, endMs: 6_000 }, + ]), + ).toContain("concat=n=2:v=0:a=1[aout]"); + }); +}); + +describe("buildEditedTrackSourceAudioFilter", () => { + it("builds a concat filtergraph that pitch-shifts via asetrate for speed changes", () => { + const filter = buildEditedTrackSourceAudioFilter( + [ + { startMs: 0, endMs: 2_000, speed: 1 }, + { startMs: 2_000, endMs: 6_000, speed: 1.5 }, + ], + 44_100, + ); + + expect(filter).toContain( + "[1:a]atrim=start=2.000:end=6.000,asetpts=PTS-STARTPTS,asetrate=66150,aresample=44100[edited_audio_1]", + ); + expect(filter).toContain("concat=n=2:v=0:a=1[aout]"); + }); + + it("returns null when the edited-track filtergraph inputs are incomplete", () => { + expect(buildEditedTrackSourceAudioFilter([], 44_100)).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter( + [{ startMs: 0, endMs: 2_000, speed: 1.5 }], + Number.NaN, + ), + ).toBeNull(); + }); +}); diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index 58dbf5ccc..fd7435faa 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -3,6 +3,9 @@ const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4; export type NativeExportEncodingMode = "fast" | "balanced" | "quality"; export type NativeVideoExportAudioMode = "none" | "copy-source" | "trim-source" | "edited-track"; +export type NativeVideoExportEditedTrackStrategy = + | "filtergraph-fast-path" + | "offline-render-fallback"; export interface NativeVideoExportStartOptions { width: number; @@ -18,10 +21,17 @@ export interface NativeVideoExportAudioSegment { endMs: number; } +export interface NativeVideoExportEditedTrackSegment extends NativeVideoExportAudioSegment { + speed: number; +} + export interface NativeVideoExportFinishOptions { audioMode?: NativeVideoExportAudioMode; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: NativeVideoExportAudioSegment[]; + editedTrackStrategy?: NativeVideoExportEditedTrackStrategy; + editedTrackSegments?: NativeVideoExportEditedTrackSegment[]; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; } @@ -162,6 +172,54 @@ export function buildTrimmedSourceAudioFilter( return filterParts.join(";"); } +export function buildEditedTrackSourceAudioFilter( + segments: NativeVideoExportEditedTrackSegment[], + sourceSampleRate: number, +): string | null { + if (segments.length === 0 || !Number.isFinite(sourceSampleRate) || sourceSampleRate <= 0) { + return null; + } + + const normalizedSourceSampleRate = Math.round(sourceSampleRate); + const filterParts: string[] = []; + const segmentLabels: string[] = []; + + segments.forEach((segment, index) => { + if (segment.endMs - segment.startMs <= 0.5) { + return; + } + + const label = `edited_audio_${index}`; + const speed = Number.isFinite(segment.speed) && segment.speed > 0 ? segment.speed : 1; + const segmentFilter = [ + `[1:a]atrim=start=${formatFfmpegSeconds(segment.startMs)}:end=${formatFfmpegSeconds(segment.endMs)}`, + "asetpts=PTS-STARTPTS", + ]; + + if (Math.abs(speed - 1) > 0.0001) { + segmentFilter.push( + `asetrate=${Math.max(1, Math.round(normalizedSourceSampleRate * speed))}`, + `aresample=${normalizedSourceSampleRate}`, + ); + } + + filterParts.push(`${segmentFilter.join(",")}[${label}]`); + segmentLabels.push(`[${label}]`); + }); + + if (segmentLabels.length === 0) { + return null; + } + + if (segmentLabels.length === 1) { + filterParts.push(`${segmentLabels[0]}anull[aout]`); + } else { + filterParts.push(`${segmentLabels.join("")}concat=n=${segmentLabels.length}:v=0:a=1[aout]`); + } + + return filterParts.join(";"); +} + /** * Builds FFmpeg arguments for a zero-copy H.264 stream export. * FFmpeg receives a pre-encoded Annex B H.264 stream on stdin (produced by the diff --git a/electron/preload.ts b/electron/preload.ts index 98562522d..a8b0e67ce 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -143,7 +143,10 @@ contextBridge.exposeInMainWorld("electronAPI", { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, @@ -186,7 +189,10 @@ contextBridge.exposeInMainWorld("electronAPI", { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, diff --git a/src/lib/exporter/editedTrackStrategy.test.ts b/src/lib/exporter/editedTrackStrategy.test.ts new file mode 100644 index 000000000..d70dd83f8 --- /dev/null +++ b/src/lib/exporter/editedTrackStrategy.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from "vitest"; +import type { AudioRegion, SpeedRegion, TrimRegion } from "@/components/video-editor/types"; +import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy"; + +const SOURCE_DURATION_MS = 20_000; +const EMPTY_TRIMS: TrimRegion[] = []; + +describe("editedTrackStrategy", () => { + it("marks single-source speed-only edits as filtergraph candidates", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions: [], + sourceAudioFallbackPaths: [], + }), + ).toBe("filtergraph-fast-path"); + }); + + it("falls back when the edited track depends on sidecar sources", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.25 }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "mic.m4a", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions: [], + sourceAudioFallbackPaths: ["mic.m4a"], + }), + ).toBe("offline-render-fallback"); + }); + + it("falls back for multi-source mixes", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions: [], + sourceAudioFallbackPaths: ["mic.m4a"], + }), + ).toBe("offline-render-fallback"); + }); + + it("falls back when audio regions require overlay mixing", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 }, + ]; + const audioRegions: AudioRegion[] = [ + { + id: "audio-1", + audioPath: "overlay.wav", + startMs: 1_000, + endMs: 4_000, + volume: 0.8, + }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions, + sourceAudioFallbackPaths: [], + }), + ).toBe("offline-render-fallback"); + }); + + it("falls back for speeds outside the conservative filtergraph window", () => { + const speedRegions = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 2.5 }, + ] as SpeedRegion[]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions: [], + sourceAudioFallbackPaths: [], + }), + ).toBe("offline-render-fallback"); + }); + + it("builds source segments that preserve trims and speed boundaries", () => { + const trimRegions: TrimRegion[] = [{ id: "trim-1", startMs: 10_000, endMs: 12_000 }]; + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 6_000, speed: 1.5 }, + { id: "speed-2", startMs: 14_000, endMs: 18_000, speed: 0.75 }, + ]; + + expect( + buildEditedTrackSourceSegments(SOURCE_DURATION_MS, trimRegions, speedRegions), + ).toEqual([ + { startMs: 0, endMs: 2_000, speed: 1 }, + { startMs: 2_000, endMs: 6_000, speed: 1.5 }, + { startMs: 6_000, endMs: 10_000, speed: 1 }, + { startMs: 12_000, endMs: 14_000, speed: 1 }, + { startMs: 14_000, endMs: 18_000, speed: 0.75 }, + { startMs: 18_000, endMs: 20_000, speed: 1 }, + ]); + }); +}); diff --git a/src/lib/exporter/editedTrackStrategy.ts b/src/lib/exporter/editedTrackStrategy.ts new file mode 100644 index 000000000..52a107839 --- /dev/null +++ b/src/lib/exporter/editedTrackStrategy.ts @@ -0,0 +1,141 @@ +import type { AudioRegion, SpeedRegion, TrimRegion } from "@/components/video-editor/types"; + +const MIN_FILTERGRAPH_SPEED = 0.5; +const MAX_FILTERGRAPH_SPEED = 2; + +export type EditedTrackStrategy = "filtergraph-fast-path" | "offline-render-fallback"; + +export interface EditedTrackSourceSegment { + startMs: number; + endMs: number; + speed: number; +} + +export interface EditedTrackStrategyInput { + primaryAudioSourcePath: string | null; + sourceDurationMs: number; + trimRegions: TrimRegion[]; + speedRegions: SpeedRegion[]; + audioRegions: AudioRegion[]; + sourceAudioFallbackPaths: string[]; +} + +function isSafeFiltergraphSpeed(speed: number): boolean { + return ( + Number.isFinite(speed) && speed >= MIN_FILTERGRAPH_SPEED && speed <= MAX_FILTERGRAPH_SPEED + ); +} + +function hasSafeFiltergraphSpeedRegions(speedRegions: SpeedRegion[]): boolean { + return speedRegions.every((region) => isSafeFiltergraphSpeed(region.speed)); +} + +function buildKeptRanges( + sourceDurationMs: number, + trimRegions: TrimRegion[], +): Array<{ startMs: number; endMs: number }> { + const sortedTrimRegions = [...trimRegions].sort((left, right) => left.startMs - right.startMs); + const keptRanges: Array<{ startMs: number; endMs: number }> = []; + let cursorMs = 0; + + for (const trimRegion of sortedTrimRegions) { + const startMs = Math.max(0, trimRegion.startMs); + const endMs = Math.min(sourceDurationMs, trimRegion.endMs); + if (endMs <= startMs) { + continue; + } + + if (startMs > cursorMs) { + keptRanges.push({ startMs: cursorMs, endMs: startMs }); + } + + cursorMs = Math.max(cursorMs, endMs); + } + + if (cursorMs < sourceDurationMs) { + keptRanges.push({ startMs: cursorMs, endMs: sourceDurationMs }); + } + + return keptRanges; +} + +export function buildEditedTrackSourceSegments( + sourceDurationMs: number, + trimRegions: TrimRegion[], + speedRegions: SpeedRegion[], +): EditedTrackSourceSegment[] { + const segments: EditedTrackSourceSegment[] = []; + const keptRanges = buildKeptRanges(sourceDurationMs, trimRegions); + + for (const keptRange of keptRanges) { + const boundaries = new Set([keptRange.startMs, keptRange.endMs]); + for (const speedRegion of speedRegions) { + const startMs = Math.max(keptRange.startMs, speedRegion.startMs); + const endMs = Math.min(keptRange.endMs, speedRegion.endMs); + if (endMs > startMs) { + boundaries.add(startMs); + boundaries.add(endMs); + } + } + + const orderedBoundaries = [...boundaries].sort((left, right) => left - right); + for (let index = 0; index < orderedBoundaries.length - 1; index += 1) { + const startMs = orderedBoundaries[index] ?? 0; + const endMs = orderedBoundaries[index + 1] ?? 0; + if (endMs - startMs <= 0.5) { + continue; + } + + const midpointMs = startMs + (endMs - startMs) / 2; + const speedRegion = speedRegions.find( + (region) => midpointMs >= region.startMs && midpointMs < region.endMs, + ); + const speed = speedRegion?.speed ?? 1; + if (!isSafeFiltergraphSpeed(speed)) { + return []; + } + + segments.push({ + startMs, + endMs, + speed, + }); + } + } + + return segments; +} + +export function classifyEditedTrackStrategy(input: EditedTrackStrategyInput): EditedTrackStrategy { + if (!input.primaryAudioSourcePath) { + return "offline-render-fallback"; + } + + if (input.audioRegions.length > 0) { + return "offline-render-fallback"; + } + + if (input.speedRegions.length === 0) { + return "offline-render-fallback"; + } + + if (!hasSafeFiltergraphSpeedRegions(input.speedRegions)) { + return "offline-render-fallback"; + } + + if (input.sourceAudioFallbackPaths.length > 0) { + return "offline-render-fallback"; + } + + if ( + buildEditedTrackSourceSegments( + input.sourceDurationMs, + input.trimRegions, + input.speedRegions, + ).length === 0 + ) { + return "offline-render-fallback"; + } + + return "filtergraph-fast-path"; +} diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 843c56ef7..15f71a583 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -15,6 +15,7 @@ import type { import { extensionHost } from "@/lib/extensions"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; import { normalizeLightningRuntimePlatform } from "./backendPolicy"; +import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy"; import { type ExportBackpressureProfile, getExportBackpressureProfile, @@ -106,6 +107,14 @@ type NativeAudioPlan = } | { audioMode: "edited-track"; + strategy: "offline-render-fallback"; + } + | { + audioMode: "edited-track"; + strategy: "filtergraph-fast-path"; + audioSourcePath: string; + audioSourceSampleRate: number; + editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>; }; const NATIVE_EXPORT_ENGINE_NAME = "Breeze"; @@ -737,11 +746,62 @@ export class ModernVideoExporter { audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 ) { - return { audioMode: "edited-track" }; + const sourceDurationMs = Math.max( + 0, + Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000), + ); + const trimRegions = this.config.trimRegions ?? []; + const strategy = + videoInfo.hasAudio && + localVideoSourcePath && + sourceAudioFallbackPaths.length === 0 && + typeof videoInfo.audioSampleRate === "number" && + Number.isFinite(videoInfo.audioSampleRate) && + videoInfo.audioSampleRate > 0 + ? classifyEditedTrackStrategy({ + primaryAudioSourcePath, + sourceDurationMs, + trimRegions, + speedRegions, + audioRegions, + sourceAudioFallbackPaths, + }) + : "offline-render-fallback"; + + if (strategy === "filtergraph-fast-path") { + const audioSourcePath = localVideoSourcePath; + const audioSourceSampleRate = videoInfo.audioSampleRate; + const editedTrackSegments = buildEditedTrackSourceSegments( + sourceDurationMs, + trimRegions, + speedRegions, + ); + if ( + audioSourcePath && + typeof audioSourceSampleRate === "number" && + editedTrackSegments.length > 0 + ) { + return { + audioMode: "edited-track", + strategy, + audioSourcePath, + audioSourceSampleRate, + editedTrackSegments, + }; + } + } + + return { + audioMode: "edited-track", + strategy: "offline-render-fallback", + }; } if (!primaryAudioSourcePath) { - return { audioMode: "edited-track" }; + return { + audioMode: "edited-track", + strategy: "offline-render-fallback", + }; } if ((this.config.trimRegions ?? []).length > 0) { @@ -949,7 +1009,10 @@ export class ModernVideoExporter { let editedAudioBuffer: ArrayBuffer | undefined; let editedAudioMimeType: string | null = null; - if (audioPlan.audioMode === "edited-track") { + if ( + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "offline-render-fallback" + ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { this.reportFinalizingProgress(this.processedFrameCount, 99, progress); @@ -976,6 +1039,8 @@ export class ModernVideoExporter { console.log(`[VideoExporter] Finalizing ${NATIVE_EXPORT_ENGINE_NAME} export`, { sessionId, audioMode: audioPlan.audioMode, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, encoderName: this.encoderName ?? "unknown", }); @@ -987,11 +1052,25 @@ export class ModernVideoExporter { audioMode: audioPlan.audioMode, audioSourcePath: audioPlan.audioMode === "copy-source" || - audioPlan.audioMode === "trim-source" + audioPlan.audioMode === "trim-source" || + (audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path") ? audioPlan.audioSourcePath : null, trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + editedTrackSegments: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.editedTrackSegments + : undefined, + audioSourceSampleRate: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.audioSourceSampleRate + : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), @@ -1041,7 +1120,10 @@ export class ModernVideoExporter { let editedAudioBuffer: ArrayBuffer | undefined; let editedAudioMimeType: string | null = null; - if (audioPlan.audioMode === "edited-track") { + if ( + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "offline-render-fallback" + ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { this.reportFinalizingProgress(this.processedFrameCount, 99, progress); @@ -1071,11 +1153,25 @@ export class ModernVideoExporter { audioMode: audioPlan.audioMode, audioSourcePath: audioPlan.audioMode === "copy-source" || - audioPlan.audioMode === "trim-source" + audioPlan.audioMode === "trim-source" || + (audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path") ? audioPlan.audioSourcePath : null, trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + editedTrackSegments: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.editedTrackSegments + : undefined, + audioSourceSampleRate: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.audioSourceSampleRate + : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index d13ff8888..6463dca20 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -17,6 +17,7 @@ export interface DecodedVideoInfo { codec: string; hasAudio: boolean; audioCodec?: string; + audioSampleRate?: number; } /** Caller must close the VideoFrame after use. */ @@ -205,6 +206,10 @@ export class StreamingVideoDecoder { codec: videoStream?.codec_string || "unknown", hasAudio: !!audioStream, audioCodec: audioStream?.codec_string, + audioSampleRate: + typeof audioStream?.sample_rate === "string" + ? Number.parseInt(audioStream.sample_rate, 10) + : undefined, }; return this.metadata; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 42f4995a3..8f6e772e9 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -13,6 +13,7 @@ import type { ZoomTransitionEasing, } from "@/components/video-editor/types"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; +import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy"; import { advanceFinalizationProgress, type FinalizationProgressWatchdog, @@ -94,6 +95,14 @@ type NativeAudioPlan = } | { audioMode: "edited-track"; + strategy: "offline-render-fallback"; + } + | { + audioMode: "edited-track"; + strategy: "filtergraph-fast-path"; + audioSourcePath: string; + audioSourceSampleRate: number; + editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>; }; export class VideoExporter { @@ -504,11 +513,62 @@ export class VideoExporter { audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 ) { - return { audioMode: "edited-track" }; + const sourceDurationMs = Math.max( + 0, + Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000), + ); + const trimRegions = this.config.trimRegions ?? []; + const strategy = + videoInfo.hasAudio && + localVideoSourcePath && + sourceAudioFallbackPaths.length === 0 && + typeof videoInfo.audioSampleRate === "number" && + Number.isFinite(videoInfo.audioSampleRate) && + videoInfo.audioSampleRate > 0 + ? classifyEditedTrackStrategy({ + primaryAudioSourcePath, + sourceDurationMs, + trimRegions, + speedRegions, + audioRegions, + sourceAudioFallbackPaths, + }) + : "offline-render-fallback"; + + if (strategy === "filtergraph-fast-path") { + const audioSourcePath = localVideoSourcePath; + const audioSourceSampleRate = videoInfo.audioSampleRate; + const editedTrackSegments = buildEditedTrackSourceSegments( + sourceDurationMs, + trimRegions, + speedRegions, + ); + if ( + audioSourcePath && + typeof audioSourceSampleRate === "number" && + editedTrackSegments.length > 0 + ) { + return { + audioMode: "edited-track", + strategy, + audioSourcePath, + audioSourceSampleRate, + editedTrackSegments, + }; + } + } + + return { + audioMode: "edited-track", + strategy: "offline-render-fallback", + }; } if (!primaryAudioSourcePath) { - return { audioMode: "edited-track" }; + return { + audioMode: "edited-track", + strategy: "offline-render-fallback", + }; } if ((this.config.trimRegions ?? []).length > 0) { @@ -709,7 +769,10 @@ export class VideoExporter { let editedAudioBuffer: ArrayBuffer | undefined; let editedAudioMimeType: string | null = null; - if (audioPlan.audioMode === "edited-track") { + if ( + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "offline-render-fallback" + ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { this.reportFinalizingProgress(totalFrames, 99, progress); @@ -741,11 +804,25 @@ export class VideoExporter { audioMode: audioPlan.audioMode, audioSourcePath: audioPlan.audioMode === "copy-source" || - audioPlan.audioMode === "trim-source" + audioPlan.audioMode === "trim-source" || + (audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path") ? audioPlan.audioSourcePath : null, trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + editedTrackSegments: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.editedTrackSegments + : undefined, + audioSourceSampleRate: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.audioSourceSampleRate + : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), @@ -790,7 +867,10 @@ export class VideoExporter { let editedAudioBuffer: ArrayBuffer | undefined; let editedAudioMimeType: string | null = null; - if (audioPlan.audioMode === "edited-track") { + if ( + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "offline-render-fallback" + ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { this.reportFinalizingProgress(totalFrames, 99, progress); @@ -820,11 +900,25 @@ export class VideoExporter { audioMode: audioPlan.audioMode, audioSourcePath: audioPlan.audioMode === "copy-source" || - audioPlan.audioMode === "trim-source" + audioPlan.audioMode === "trim-source" || + (audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path") ? audioPlan.audioSourcePath : null, trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + editedTrackSegments: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.editedTrackSegments + : undefined, + audioSourceSampleRate: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.audioSourceSampleRate + : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), From 2b19493bb7ca1914d6b38464ae2fcbb12cadd3c2 Mon Sep 17 00:00:00 2001 From: wiiiii123 Date: Tue, 21 Apr 2026 00:53:32 +0700 Subject: [PATCH 2/5] fix(export): validate edited-track fast-path inputs --- electron/ipc/nativeVideoExport.test.ts | 15 +++++++ electron/ipc/nativeVideoExport.ts | 28 +++++++++++-- src/lib/exporter/editedTrackStrategy.test.ts | 25 ++++++++++++ src/lib/exporter/editedTrackStrategy.ts | 41 ++++++++++++++++++-- 4 files changed, 103 insertions(+), 6 deletions(-) diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts index 1d0630d98..eefbad037 100644 --- a/electron/ipc/nativeVideoExport.test.ts +++ b/electron/ipc/nativeVideoExport.test.ts @@ -40,4 +40,19 @@ describe("buildEditedTrackSourceAudioFilter", () => { ), ).toBeNull(); }); + + it("returns null when the edited-track segments are malformed", () => { + expect( + buildEditedTrackSourceAudioFilter( + [{ startMs: Number.NaN, endMs: 2_000, speed: 1.5 }], + 44_100, + ), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: 0 }], 44_100), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: 1 }], 0.4), + ).toBeNull(); + }); }); diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index fd7435faa..0f59eddf5 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -181,24 +181,46 @@ export function buildEditedTrackSourceAudioFilter( } const normalizedSourceSampleRate = Math.round(sourceSampleRate); + if (normalizedSourceSampleRate < 1) { + return null; + } + const filterParts: string[] = []; const segmentLabels: string[] = []; + let hasInvalidSegment = false; segments.forEach((segment, index) => { + if (!Number.isFinite(segment.startMs) || !Number.isFinite(segment.endMs)) { + hasInvalidSegment = true; + return; + } + if (segment.endMs - segment.startMs <= 0.5) { + hasInvalidSegment = true; return; } const label = `edited_audio_${index}`; - const speed = Number.isFinite(segment.speed) && segment.speed > 0 ? segment.speed : 1; + const speed = segment.speed; + if (!Number.isFinite(speed) || speed <= 0) { + hasInvalidSegment = true; + return; + } + const segmentFilter = [ `[1:a]atrim=start=${formatFfmpegSeconds(segment.startMs)}:end=${formatFfmpegSeconds(segment.endMs)}`, "asetpts=PTS-STARTPTS", ]; if (Math.abs(speed - 1) > 0.0001) { + const adjustedSampleRate = Math.round(normalizedSourceSampleRate * speed); + if (adjustedSampleRate < 1) { + hasInvalidSegment = true; + return; + } + segmentFilter.push( - `asetrate=${Math.max(1, Math.round(normalizedSourceSampleRate * speed))}`, + `asetrate=${adjustedSampleRate}`, `aresample=${normalizedSourceSampleRate}`, ); } @@ -207,7 +229,7 @@ export function buildEditedTrackSourceAudioFilter( segmentLabels.push(`[${label}]`); }); - if (segmentLabels.length === 0) { + if (hasInvalidSegment || segmentLabels.length === 0) { return null; } diff --git a/src/lib/exporter/editedTrackStrategy.test.ts b/src/lib/exporter/editedTrackStrategy.test.ts index d70dd83f8..9201b5332 100644 --- a/src/lib/exporter/editedTrackStrategy.test.ts +++ b/src/lib/exporter/editedTrackStrategy.test.ts @@ -100,6 +100,23 @@ describe("editedTrackStrategy", () => { ).toBe("offline-render-fallback"); }); + it("falls back when source duration or speed-region bounds are invalid", () => { + const invalidBounds: SpeedRegion[] = [ + { id: "speed-1", startMs: Number.NaN, endMs: 10_000, speed: 1.5 }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: Number.POSITIVE_INFINITY, + trimRegions: EMPTY_TRIMS, + speedRegions: invalidBounds, + audioRegions: [], + sourceAudioFallbackPaths: [], + }), + ).toBe("offline-render-fallback"); + }); + it("builds source segments that preserve trims and speed boundaries", () => { const trimRegions: TrimRegion[] = [{ id: "trim-1", startMs: 10_000, endMs: 12_000 }]; const speedRegions: SpeedRegion[] = [ @@ -118,4 +135,12 @@ describe("editedTrackStrategy", () => { { startMs: 18_000, endMs: 20_000, speed: 1 }, ]); }); + + it("returns no source segments when speed-region bounds are invalid", () => { + expect( + buildEditedTrackSourceSegments(SOURCE_DURATION_MS, EMPTY_TRIMS, [ + { id: "speed-1", startMs: 15_000, endMs: 10_000, speed: 1.5 }, + ]), + ).toEqual([]); + }); }); diff --git a/src/lib/exporter/editedTrackStrategy.ts b/src/lib/exporter/editedTrackStrategy.ts index 52a107839..5bee89ee3 100644 --- a/src/lib/exporter/editedTrackStrategy.ts +++ b/src/lib/exporter/editedTrackStrategy.ts @@ -26,8 +26,29 @@ function isSafeFiltergraphSpeed(speed: number): boolean { ); } -function hasSafeFiltergraphSpeedRegions(speedRegions: SpeedRegion[]): boolean { - return speedRegions.every((region) => isSafeFiltergraphSpeed(region.speed)); +function hasFiniteTimelineRange(startMs: number, endMs: number, sourceDurationMs: number): boolean { + return ( + Number.isFinite(startMs) && + Number.isFinite(endMs) && + startMs >= 0 && + endMs > startMs && + endMs <= sourceDurationMs + ); +} + +function hasSafeFiltergraphSpeedRegions( + speedRegions: SpeedRegion[], + sourceDurationMs: number, +): boolean { + if (!Number.isFinite(sourceDurationMs) || sourceDurationMs <= 0) { + return false; + } + + return speedRegions.every( + (region) => + hasFiniteTimelineRange(region.startMs, region.endMs, sourceDurationMs) && + isSafeFiltergraphSpeed(region.speed), + ); } function buildKeptRanges( @@ -64,6 +85,20 @@ export function buildEditedTrackSourceSegments( trimRegions: TrimRegion[], speedRegions: SpeedRegion[], ): EditedTrackSourceSegment[] { + if (!Number.isFinite(sourceDurationMs) || sourceDurationMs <= 0) { + return []; + } + + if ( + speedRegions.some( + (region) => + !hasFiniteTimelineRange(region.startMs, region.endMs, sourceDurationMs) || + !isSafeFiltergraphSpeed(region.speed), + ) + ) { + return []; + } + const segments: EditedTrackSourceSegment[] = []; const keptRanges = buildKeptRanges(sourceDurationMs, trimRegions); @@ -119,7 +154,7 @@ export function classifyEditedTrackStrategy(input: EditedTrackStrategyInput): Ed return "offline-render-fallback"; } - if (!hasSafeFiltergraphSpeedRegions(input.speedRegions)) { + if (!hasSafeFiltergraphSpeedRegions(input.speedRegions, input.sourceDurationMs)) { return "offline-render-fallback"; } From 3b2922cfbbefc55f7c5c533306d6aa9d128a93cd Mon Sep 17 00:00:00 2001 From: wiiiii123 Date: Tue, 21 Apr 2026 01:09:17 +0700 Subject: [PATCH 3/5] fix(export): harden edited-track filter validation --- electron/ipc/nativeVideoExport.test.ts | 9 +++++++++ electron/ipc/nativeVideoExport.ts | 9 +++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts index eefbad037..aa213073c 100644 --- a/electron/ipc/nativeVideoExport.test.ts +++ b/electron/ipc/nativeVideoExport.test.ts @@ -54,5 +54,14 @@ describe("buildEditedTrackSourceAudioFilter", () => { expect( buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: 1 }], 0.4), ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter([{ startMs: -100, endMs: 2_000, speed: 1 }], 44_100), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter( + [{ startMs: 0, endMs: 2_000, speed: Number.MAX_SAFE_INTEGER }], + 44_100, + ), + ).toBeNull(); }); }); diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index 0f59eddf5..5ca8fa081 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -190,7 +190,12 @@ export function buildEditedTrackSourceAudioFilter( let hasInvalidSegment = false; segments.forEach((segment, index) => { - if (!Number.isFinite(segment.startMs) || !Number.isFinite(segment.endMs)) { + if ( + !Number.isFinite(segment.startMs) || + !Number.isFinite(segment.endMs) || + segment.startMs < 0 || + segment.endMs < 0 + ) { hasInvalidSegment = true; return; } @@ -214,7 +219,7 @@ export function buildEditedTrackSourceAudioFilter( if (Math.abs(speed - 1) > 0.0001) { const adjustedSampleRate = Math.round(normalizedSourceSampleRate * speed); - if (adjustedSampleRate < 1) { + if (!Number.isSafeInteger(adjustedSampleRate) || adjustedSampleRate < 1) { hasInvalidSegment = true; return; } From aec1dd8419dc3ee6ca03af7b8884253bc8ef8da7 Mon Sep 17 00:00:00 2001 From: wiiiii123 Date: Tue, 21 Apr 2026 01:21:22 +0700 Subject: [PATCH 4/5] test(export): tighten native filtergraph assertions --- electron/ipc/nativeVideoExport.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts index aa213073c..14f191030 100644 --- a/electron/ipc/nativeVideoExport.test.ts +++ b/electron/ipc/nativeVideoExport.test.ts @@ -11,7 +11,11 @@ describe("buildTrimmedSourceAudioFilter", () => { { startMs: 0, endMs: 2_000 }, { startMs: 4_000, endMs: 6_000 }, ]), - ).toContain("concat=n=2:v=0:a=1[aout]"); + ).toBe( + "[1:a]atrim=start=0.000:end=2.000,asetpts=PTS-STARTPTS[trimmed_audio_0];" + + "[1:a]atrim=start=4.000:end=6.000,asetpts=PTS-STARTPTS[trimmed_audio_1];" + + "[trimmed_audio_0][trimmed_audio_1]concat=n=2:v=0:a=1[aout]", + ); }); }); @@ -25,10 +29,11 @@ describe("buildEditedTrackSourceAudioFilter", () => { 44_100, ); - expect(filter).toContain( - "[1:a]atrim=start=2.000:end=6.000,asetpts=PTS-STARTPTS,asetrate=66150,aresample=44100[edited_audio_1]", + expect(filter).toBe( + "[1:a]atrim=start=0.000:end=2.000,asetpts=PTS-STARTPTS[edited_audio_0];" + + "[1:a]atrim=start=2.000:end=6.000,asetpts=PTS-STARTPTS,asetrate=66150,aresample=44100[edited_audio_1];" + + "[edited_audio_0][edited_audio_1]concat=n=2:v=0:a=1[aout]", ); - expect(filter).toContain("concat=n=2:v=0:a=1[aout]"); }); it("returns null when the edited-track filtergraph inputs are incomplete", () => { From 383bde994e18a0d4326a42922a50b98584f7bc3f Mon Sep 17 00:00:00 2001 From: wiiiii123 Date: Tue, 21 Apr 2026 01:36:19 +0700 Subject: [PATCH 5/5] test(export): cover slowdown filtergraph inputs --- electron/ipc/nativeVideoExport.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts index 14f191030..855e3c07f 100644 --- a/electron/ipc/nativeVideoExport.test.ts +++ b/electron/ipc/nativeVideoExport.test.ts @@ -36,6 +36,18 @@ describe("buildEditedTrackSourceAudioFilter", () => { ); }); + it("builds a filtergraph for slowdown segments by lowering then resampling the source rate", () => { + const filter = buildEditedTrackSourceAudioFilter( + [{ startMs: 0, endMs: 2_000, speed: 0.5 }], + 44_100, + ); + + expect(filter).toBe( + "[1:a]atrim=start=0.000:end=2.000,asetpts=PTS-STARTPTS,asetrate=22050,aresample=44100[edited_audio_0];" + + "[edited_audio_0]anull[aout]", + ); + }); + it("returns null when the edited-track filtergraph inputs are incomplete", () => { expect(buildEditedTrackSourceAudioFilter([], 44_100)).toBeNull(); expect( @@ -56,6 +68,15 @@ describe("buildEditedTrackSourceAudioFilter", () => { expect( buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: 0 }], 44_100), ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: -1 }], 44_100), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter( + [{ startMs: 0, endMs: 2_000, speed: Number.NaN }], + 44_100, + ), + ).toBeNull(); expect( buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: 1 }], 0.4), ).toBeNull();