diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index f37eae2b1..5012dbe5c 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -194,7 +194,10 @@ interface Window { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, @@ -213,7 +216,10 @@ interface Window { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, diff --git a/electron/ipc/export/native-video.ts b/electron/ipc/export/native-video.ts index 77552cbb6..a92053698 100644 --- a/electron/ipc/export/native-video.ts +++ b/electron/ipc/export/native-video.ts @@ -14,6 +14,7 @@ import type { NativeVideoExportFinishOptions, } from "../nativeVideoExport"; import { + buildEditedTrackSourceAudioFilter, buildNativeVideoExportArgs, buildTrimmedSourceAudioFilter, getEditedAudioExtension, @@ -387,8 +388,10 @@ export async function muxNativeVideoExportAudio( const metrics: NativeVideoAudioMuxMetrics = {}; const tempArtifacts: string[] = []; let audioInputPath = options.audioSourcePath ?? null; + const useEditedTrackFiltergraph = + audioMode === "edited-track" && options.editedTrackStrategy === "filtergraph-fast-path"; - if (audioMode === "edited-track") { + if (audioMode === "edited-track" && !useEditedTrackFiltergraph) { if (!options.editedAudioData) { throw new Error("Edited audio data is missing for native export"); } @@ -435,6 +438,15 @@ export async function muxNativeVideoExportAudio( } else { args.push("-map", "0:v:0", "-map", "1:a:0"); } + } else if (useEditedTrackFiltergraph) { + const filter = buildEditedTrackSourceAudioFilter( + options.editedTrackSegments ?? [], + options.audioSourceSampleRate ?? 0, + ); + if (!filter) { + throw new Error("Edited-track filtergraph inputs are incomplete for native export"); + } + args.push("-filter_complex", filter, "-map", "0:v:0", "-map", "[aout]"); } else { args.push("-map", "0:v:0", "-map", "1:a:0"); } diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts new file mode 100644 index 000000000..855e3c07f --- /dev/null +++ b/electron/ipc/nativeVideoExport.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { + buildEditedTrackSourceAudioFilter, + buildTrimmedSourceAudioFilter, +} from "./nativeVideoExport"; + +describe("buildTrimmedSourceAudioFilter", () => { + it("concatenates trimmed source segments into a single output label", () => { + expect( + buildTrimmedSourceAudioFilter([ + { startMs: 0, endMs: 2_000 }, + { startMs: 4_000, endMs: 6_000 }, + ]), + ).toBe( + "[1:a]atrim=start=0.000:end=2.000,asetpts=PTS-STARTPTS[trimmed_audio_0];" + + "[1:a]atrim=start=4.000:end=6.000,asetpts=PTS-STARTPTS[trimmed_audio_1];" + + "[trimmed_audio_0][trimmed_audio_1]concat=n=2:v=0:a=1[aout]", + ); + }); +}); + +describe("buildEditedTrackSourceAudioFilter", () => { + it("builds a concat filtergraph that pitch-shifts via asetrate for speed changes", () => { + const filter = buildEditedTrackSourceAudioFilter( + [ + { startMs: 0, endMs: 2_000, speed: 1 }, + { startMs: 2_000, endMs: 6_000, speed: 1.5 }, + ], + 44_100, + ); + + expect(filter).toBe( + "[1:a]atrim=start=0.000:end=2.000,asetpts=PTS-STARTPTS[edited_audio_0];" + + "[1:a]atrim=start=2.000:end=6.000,asetpts=PTS-STARTPTS,asetrate=66150,aresample=44100[edited_audio_1];" + + "[edited_audio_0][edited_audio_1]concat=n=2:v=0:a=1[aout]", + ); + }); + + it("builds a filtergraph for slowdown segments by lowering then resampling the source rate", () => { + const filter = buildEditedTrackSourceAudioFilter( + [{ startMs: 0, endMs: 2_000, speed: 0.5 }], + 44_100, + ); + + expect(filter).toBe( + "[1:a]atrim=start=0.000:end=2.000,asetpts=PTS-STARTPTS,asetrate=22050,aresample=44100[edited_audio_0];" + + "[edited_audio_0]anull[aout]", + ); + }); + + it("returns null when the edited-track filtergraph inputs are incomplete", () => { + expect(buildEditedTrackSourceAudioFilter([], 44_100)).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter( + [{ startMs: 0, endMs: 2_000, speed: 1.5 }], + Number.NaN, + ), + ).toBeNull(); + }); + + it("returns null when the edited-track segments are malformed", () => { + expect( + buildEditedTrackSourceAudioFilter( + [{ startMs: Number.NaN, endMs: 2_000, speed: 1.5 }], + 44_100, + ), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: 0 }], 44_100), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: -1 }], 44_100), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter( + [{ startMs: 0, endMs: 2_000, speed: Number.NaN }], + 44_100, + ), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter([{ startMs: 0, endMs: 2_000, speed: 1 }], 0.4), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter([{ startMs: -100, endMs: 2_000, speed: 1 }], 44_100), + ).toBeNull(); + expect( + buildEditedTrackSourceAudioFilter( + [{ startMs: 0, endMs: 2_000, speed: Number.MAX_SAFE_INTEGER }], + 44_100, + ), + ).toBeNull(); + }); +}); diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index 58dbf5ccc..5ca8fa081 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -3,6 +3,9 @@ const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4; export type NativeExportEncodingMode = "fast" | "balanced" | "quality"; export type NativeVideoExportAudioMode = "none" | "copy-source" | "trim-source" | "edited-track"; +export type NativeVideoExportEditedTrackStrategy = + | "filtergraph-fast-path" + | "offline-render-fallback"; export interface NativeVideoExportStartOptions { width: number; @@ -18,10 +21,17 @@ export interface NativeVideoExportAudioSegment { endMs: number; } +export interface NativeVideoExportEditedTrackSegment extends NativeVideoExportAudioSegment { + speed: number; +} + export interface NativeVideoExportFinishOptions { audioMode?: NativeVideoExportAudioMode; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: NativeVideoExportAudioSegment[]; + editedTrackStrategy?: NativeVideoExportEditedTrackStrategy; + editedTrackSegments?: NativeVideoExportEditedTrackSegment[]; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; } @@ -162,6 +172,81 @@ export function buildTrimmedSourceAudioFilter( return filterParts.join(";"); } +export function buildEditedTrackSourceAudioFilter( + segments: NativeVideoExportEditedTrackSegment[], + sourceSampleRate: number, +): string | null { + if (segments.length === 0 || !Number.isFinite(sourceSampleRate) || sourceSampleRate <= 0) { + return null; + } + + const normalizedSourceSampleRate = Math.round(sourceSampleRate); + if (normalizedSourceSampleRate < 1) { + return null; + } + + const filterParts: string[] = []; + const segmentLabels: string[] = []; + let hasInvalidSegment = false; + + segments.forEach((segment, index) => { + if ( + !Number.isFinite(segment.startMs) || + !Number.isFinite(segment.endMs) || + segment.startMs < 0 || + segment.endMs < 0 + ) { + hasInvalidSegment = true; + return; + } + + if (segment.endMs - segment.startMs <= 0.5) { + hasInvalidSegment = true; + return; + } + + const label = `edited_audio_${index}`; + const speed = segment.speed; + if (!Number.isFinite(speed) || speed <= 0) { + hasInvalidSegment = true; + return; + } + + const segmentFilter = [ + `[1:a]atrim=start=${formatFfmpegSeconds(segment.startMs)}:end=${formatFfmpegSeconds(segment.endMs)}`, + "asetpts=PTS-STARTPTS", + ]; + + if (Math.abs(speed - 1) > 0.0001) { + const adjustedSampleRate = Math.round(normalizedSourceSampleRate * speed); + if (!Number.isSafeInteger(adjustedSampleRate) || adjustedSampleRate < 1) { + hasInvalidSegment = true; + return; + } + + segmentFilter.push( + `asetrate=${adjustedSampleRate}`, + `aresample=${normalizedSourceSampleRate}`, + ); + } + + filterParts.push(`${segmentFilter.join(",")}[${label}]`); + segmentLabels.push(`[${label}]`); + }); + + if (hasInvalidSegment || segmentLabels.length === 0) { + return null; + } + + if (segmentLabels.length === 1) { + filterParts.push(`${segmentLabels[0]}anull[aout]`); + } else { + filterParts.push(`${segmentLabels.join("")}concat=n=${segmentLabels.length}:v=0:a=1[aout]`); + } + + return filterParts.join(";"); +} + /** * Builds FFmpeg arguments for a zero-copy H.264 stream export. * FFmpeg receives a pre-encoded Annex B H.264 stream on stdin (produced by the diff --git a/electron/preload.ts b/electron/preload.ts index 98562522d..a8b0e67ce 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -143,7 +143,10 @@ contextBridge.exposeInMainWorld("electronAPI", { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, @@ -186,7 +189,10 @@ contextBridge.exposeInMainWorld("electronAPI", { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, diff --git a/src/lib/exporter/editedTrackStrategy.test.ts b/src/lib/exporter/editedTrackStrategy.test.ts new file mode 100644 index 000000000..9201b5332 --- /dev/null +++ b/src/lib/exporter/editedTrackStrategy.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import type { AudioRegion, SpeedRegion, TrimRegion } from "@/components/video-editor/types"; +import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy"; + +const SOURCE_DURATION_MS = 20_000; +const EMPTY_TRIMS: TrimRegion[] = []; + +describe("editedTrackStrategy", () => { + it("marks single-source speed-only edits as filtergraph candidates", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions: [], + sourceAudioFallbackPaths: [], + }), + ).toBe("filtergraph-fast-path"); + }); + + it("falls back when the edited track depends on sidecar sources", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.25 }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "mic.m4a", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions: [], + sourceAudioFallbackPaths: ["mic.m4a"], + }), + ).toBe("offline-render-fallback"); + }); + + it("falls back for multi-source mixes", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions: [], + sourceAudioFallbackPaths: ["mic.m4a"], + }), + ).toBe("offline-render-fallback"); + }); + + it("falls back when audio regions require overlay mixing", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 }, + ]; + const audioRegions: AudioRegion[] = [ + { + id: "audio-1", + audioPath: "overlay.wav", + startMs: 1_000, + endMs: 4_000, + volume: 0.8, + }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions, + sourceAudioFallbackPaths: [], + }), + ).toBe("offline-render-fallback"); + }); + + it("falls back for speeds outside the conservative filtergraph window", () => { + const speedRegions = [ + { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 2.5 }, + ] as SpeedRegion[]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: SOURCE_DURATION_MS, + trimRegions: EMPTY_TRIMS, + speedRegions, + audioRegions: [], + sourceAudioFallbackPaths: [], + }), + ).toBe("offline-render-fallback"); + }); + + it("falls back when source duration or speed-region bounds are invalid", () => { + const invalidBounds: SpeedRegion[] = [ + { id: "speed-1", startMs: Number.NaN, endMs: 10_000, speed: 1.5 }, + ]; + + expect( + classifyEditedTrackStrategy({ + primaryAudioSourcePath: "recording.mp4", + sourceDurationMs: Number.POSITIVE_INFINITY, + trimRegions: EMPTY_TRIMS, + speedRegions: invalidBounds, + audioRegions: [], + sourceAudioFallbackPaths: [], + }), + ).toBe("offline-render-fallback"); + }); + + it("builds source segments that preserve trims and speed boundaries", () => { + const trimRegions: TrimRegion[] = [{ id: "trim-1", startMs: 10_000, endMs: 12_000 }]; + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 2_000, endMs: 6_000, speed: 1.5 }, + { id: "speed-2", startMs: 14_000, endMs: 18_000, speed: 0.75 }, + ]; + + expect( + buildEditedTrackSourceSegments(SOURCE_DURATION_MS, trimRegions, speedRegions), + ).toEqual([ + { startMs: 0, endMs: 2_000, speed: 1 }, + { startMs: 2_000, endMs: 6_000, speed: 1.5 }, + { startMs: 6_000, endMs: 10_000, speed: 1 }, + { startMs: 12_000, endMs: 14_000, speed: 1 }, + { startMs: 14_000, endMs: 18_000, speed: 0.75 }, + { startMs: 18_000, endMs: 20_000, speed: 1 }, + ]); + }); + + it("returns no source segments when speed-region bounds are invalid", () => { + expect( + buildEditedTrackSourceSegments(SOURCE_DURATION_MS, EMPTY_TRIMS, [ + { id: "speed-1", startMs: 15_000, endMs: 10_000, speed: 1.5 }, + ]), + ).toEqual([]); + }); +}); diff --git a/src/lib/exporter/editedTrackStrategy.ts b/src/lib/exporter/editedTrackStrategy.ts new file mode 100644 index 000000000..5bee89ee3 --- /dev/null +++ b/src/lib/exporter/editedTrackStrategy.ts @@ -0,0 +1,176 @@ +import type { AudioRegion, SpeedRegion, TrimRegion } from "@/components/video-editor/types"; + +const MIN_FILTERGRAPH_SPEED = 0.5; +const MAX_FILTERGRAPH_SPEED = 2; + +export type EditedTrackStrategy = "filtergraph-fast-path" | "offline-render-fallback"; + +export interface EditedTrackSourceSegment { + startMs: number; + endMs: number; + speed: number; +} + +export interface EditedTrackStrategyInput { + primaryAudioSourcePath: string | null; + sourceDurationMs: number; + trimRegions: TrimRegion[]; + speedRegions: SpeedRegion[]; + audioRegions: AudioRegion[]; + sourceAudioFallbackPaths: string[]; +} + +function isSafeFiltergraphSpeed(speed: number): boolean { + return ( + Number.isFinite(speed) && speed >= MIN_FILTERGRAPH_SPEED && speed <= MAX_FILTERGRAPH_SPEED + ); +} + +function hasFiniteTimelineRange(startMs: number, endMs: number, sourceDurationMs: number): boolean { + return ( + Number.isFinite(startMs) && + Number.isFinite(endMs) && + startMs >= 0 && + endMs > startMs && + endMs <= sourceDurationMs + ); +} + +function hasSafeFiltergraphSpeedRegions( + speedRegions: SpeedRegion[], + sourceDurationMs: number, +): boolean { + if (!Number.isFinite(sourceDurationMs) || sourceDurationMs <= 0) { + return false; + } + + return speedRegions.every( + (region) => + hasFiniteTimelineRange(region.startMs, region.endMs, sourceDurationMs) && + isSafeFiltergraphSpeed(region.speed), + ); +} + +function buildKeptRanges( + sourceDurationMs: number, + trimRegions: TrimRegion[], +): Array<{ startMs: number; endMs: number }> { + const sortedTrimRegions = [...trimRegions].sort((left, right) => left.startMs - right.startMs); + const keptRanges: Array<{ startMs: number; endMs: number }> = []; + let cursorMs = 0; + + for (const trimRegion of sortedTrimRegions) { + const startMs = Math.max(0, trimRegion.startMs); + const endMs = Math.min(sourceDurationMs, trimRegion.endMs); + if (endMs <= startMs) { + continue; + } + + if (startMs > cursorMs) { + keptRanges.push({ startMs: cursorMs, endMs: startMs }); + } + + cursorMs = Math.max(cursorMs, endMs); + } + + if (cursorMs < sourceDurationMs) { + keptRanges.push({ startMs: cursorMs, endMs: sourceDurationMs }); + } + + return keptRanges; +} + +export function buildEditedTrackSourceSegments( + sourceDurationMs: number, + trimRegions: TrimRegion[], + speedRegions: SpeedRegion[], +): EditedTrackSourceSegment[] { + if (!Number.isFinite(sourceDurationMs) || sourceDurationMs <= 0) { + return []; + } + + if ( + speedRegions.some( + (region) => + !hasFiniteTimelineRange(region.startMs, region.endMs, sourceDurationMs) || + !isSafeFiltergraphSpeed(region.speed), + ) + ) { + return []; + } + + const segments: EditedTrackSourceSegment[] = []; + const keptRanges = buildKeptRanges(sourceDurationMs, trimRegions); + + for (const keptRange of keptRanges) { + const boundaries = new Set([keptRange.startMs, keptRange.endMs]); + for (const speedRegion of speedRegions) { + const startMs = Math.max(keptRange.startMs, speedRegion.startMs); + const endMs = Math.min(keptRange.endMs, speedRegion.endMs); + if (endMs > startMs) { + boundaries.add(startMs); + boundaries.add(endMs); + } + } + + const orderedBoundaries = [...boundaries].sort((left, right) => left - right); + for (let index = 0; index < orderedBoundaries.length - 1; index += 1) { + const startMs = orderedBoundaries[index] ?? 0; + const endMs = orderedBoundaries[index + 1] ?? 0; + if (endMs - startMs <= 0.5) { + continue; + } + + const midpointMs = startMs + (endMs - startMs) / 2; + const speedRegion = speedRegions.find( + (region) => midpointMs >= region.startMs && midpointMs < region.endMs, + ); + const speed = speedRegion?.speed ?? 1; + if (!isSafeFiltergraphSpeed(speed)) { + return []; + } + + segments.push({ + startMs, + endMs, + speed, + }); + } + } + + return segments; +} + +export function classifyEditedTrackStrategy(input: EditedTrackStrategyInput): EditedTrackStrategy { + if (!input.primaryAudioSourcePath) { + return "offline-render-fallback"; + } + + if (input.audioRegions.length > 0) { + return "offline-render-fallback"; + } + + if (input.speedRegions.length === 0) { + return "offline-render-fallback"; + } + + if (!hasSafeFiltergraphSpeedRegions(input.speedRegions, input.sourceDurationMs)) { + return "offline-render-fallback"; + } + + if (input.sourceAudioFallbackPaths.length > 0) { + return "offline-render-fallback"; + } + + if ( + buildEditedTrackSourceSegments( + input.sourceDurationMs, + input.trimRegions, + input.speedRegions, + ).length === 0 + ) { + return "offline-render-fallback"; + } + + return "filtergraph-fast-path"; +} diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 843c56ef7..15f71a583 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -15,6 +15,7 @@ import type { import { extensionHost } from "@/lib/extensions"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; import { normalizeLightningRuntimePlatform } from "./backendPolicy"; +import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy"; import { type ExportBackpressureProfile, getExportBackpressureProfile, @@ -106,6 +107,14 @@ type NativeAudioPlan = } | { audioMode: "edited-track"; + strategy: "offline-render-fallback"; + } + | { + audioMode: "edited-track"; + strategy: "filtergraph-fast-path"; + audioSourcePath: string; + audioSourceSampleRate: number; + editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>; }; const NATIVE_EXPORT_ENGINE_NAME = "Breeze"; @@ -737,11 +746,62 @@ export class ModernVideoExporter { audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 ) { - return { audioMode: "edited-track" }; + const sourceDurationMs = Math.max( + 0, + Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000), + ); + const trimRegions = this.config.trimRegions ?? []; + const strategy = + videoInfo.hasAudio && + localVideoSourcePath && + sourceAudioFallbackPaths.length === 0 && + typeof videoInfo.audioSampleRate === "number" && + Number.isFinite(videoInfo.audioSampleRate) && + videoInfo.audioSampleRate > 0 + ? classifyEditedTrackStrategy({ + primaryAudioSourcePath, + sourceDurationMs, + trimRegions, + speedRegions, + audioRegions, + sourceAudioFallbackPaths, + }) + : "offline-render-fallback"; + + if (strategy === "filtergraph-fast-path") { + const audioSourcePath = localVideoSourcePath; + const audioSourceSampleRate = videoInfo.audioSampleRate; + const editedTrackSegments = buildEditedTrackSourceSegments( + sourceDurationMs, + trimRegions, + speedRegions, + ); + if ( + audioSourcePath && + typeof audioSourceSampleRate === "number" && + editedTrackSegments.length > 0 + ) { + return { + audioMode: "edited-track", + strategy, + audioSourcePath, + audioSourceSampleRate, + editedTrackSegments, + }; + } + } + + return { + audioMode: "edited-track", + strategy: "offline-render-fallback", + }; } if (!primaryAudioSourcePath) { - return { audioMode: "edited-track" }; + return { + audioMode: "edited-track", + strategy: "offline-render-fallback", + }; } if ((this.config.trimRegions ?? []).length > 0) { @@ -949,7 +1009,10 @@ export class ModernVideoExporter { let editedAudioBuffer: ArrayBuffer | undefined; let editedAudioMimeType: string | null = null; - if (audioPlan.audioMode === "edited-track") { + if ( + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "offline-render-fallback" + ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { this.reportFinalizingProgress(this.processedFrameCount, 99, progress); @@ -976,6 +1039,8 @@ export class ModernVideoExporter { console.log(`[VideoExporter] Finalizing ${NATIVE_EXPORT_ENGINE_NAME} export`, { sessionId, audioMode: audioPlan.audioMode, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, encoderName: this.encoderName ?? "unknown", }); @@ -987,11 +1052,25 @@ export class ModernVideoExporter { audioMode: audioPlan.audioMode, audioSourcePath: audioPlan.audioMode === "copy-source" || - audioPlan.audioMode === "trim-source" + audioPlan.audioMode === "trim-source" || + (audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path") ? audioPlan.audioSourcePath : null, trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + editedTrackSegments: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.editedTrackSegments + : undefined, + audioSourceSampleRate: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.audioSourceSampleRate + : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), @@ -1041,7 +1120,10 @@ export class ModernVideoExporter { let editedAudioBuffer: ArrayBuffer | undefined; let editedAudioMimeType: string | null = null; - if (audioPlan.audioMode === "edited-track") { + if ( + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "offline-render-fallback" + ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { this.reportFinalizingProgress(this.processedFrameCount, 99, progress); @@ -1071,11 +1153,25 @@ export class ModernVideoExporter { audioMode: audioPlan.audioMode, audioSourcePath: audioPlan.audioMode === "copy-source" || - audioPlan.audioMode === "trim-source" + audioPlan.audioMode === "trim-source" || + (audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path") ? audioPlan.audioSourcePath : null, trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + editedTrackSegments: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.editedTrackSegments + : undefined, + audioSourceSampleRate: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.audioSourceSampleRate + : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index d13ff8888..6463dca20 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -17,6 +17,7 @@ export interface DecodedVideoInfo { codec: string; hasAudio: boolean; audioCodec?: string; + audioSampleRate?: number; } /** Caller must close the VideoFrame after use. */ @@ -205,6 +206,10 @@ export class StreamingVideoDecoder { codec: videoStream?.codec_string || "unknown", hasAudio: !!audioStream, audioCodec: audioStream?.codec_string, + audioSampleRate: + typeof audioStream?.sample_rate === "string" + ? Number.parseInt(audioStream.sample_rate, 10) + : undefined, }; return this.metadata; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 42f4995a3..8f6e772e9 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -13,6 +13,7 @@ import type { ZoomTransitionEasing, } from "@/components/video-editor/types"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; +import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy"; import { advanceFinalizationProgress, type FinalizationProgressWatchdog, @@ -94,6 +95,14 @@ type NativeAudioPlan = } | { audioMode: "edited-track"; + strategy: "offline-render-fallback"; + } + | { + audioMode: "edited-track"; + strategy: "filtergraph-fast-path"; + audioSourcePath: string; + audioSourceSampleRate: number; + editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>; }; export class VideoExporter { @@ -504,11 +513,62 @@ export class VideoExporter { audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 ) { - return { audioMode: "edited-track" }; + const sourceDurationMs = Math.max( + 0, + Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000), + ); + const trimRegions = this.config.trimRegions ?? []; + const strategy = + videoInfo.hasAudio && + localVideoSourcePath && + sourceAudioFallbackPaths.length === 0 && + typeof videoInfo.audioSampleRate === "number" && + Number.isFinite(videoInfo.audioSampleRate) && + videoInfo.audioSampleRate > 0 + ? classifyEditedTrackStrategy({ + primaryAudioSourcePath, + sourceDurationMs, + trimRegions, + speedRegions, + audioRegions, + sourceAudioFallbackPaths, + }) + : "offline-render-fallback"; + + if (strategy === "filtergraph-fast-path") { + const audioSourcePath = localVideoSourcePath; + const audioSourceSampleRate = videoInfo.audioSampleRate; + const editedTrackSegments = buildEditedTrackSourceSegments( + sourceDurationMs, + trimRegions, + speedRegions, + ); + if ( + audioSourcePath && + typeof audioSourceSampleRate === "number" && + editedTrackSegments.length > 0 + ) { + return { + audioMode: "edited-track", + strategy, + audioSourcePath, + audioSourceSampleRate, + editedTrackSegments, + }; + } + } + + return { + audioMode: "edited-track", + strategy: "offline-render-fallback", + }; } if (!primaryAudioSourcePath) { - return { audioMode: "edited-track" }; + return { + audioMode: "edited-track", + strategy: "offline-render-fallback", + }; } if ((this.config.trimRegions ?? []).length > 0) { @@ -709,7 +769,10 @@ export class VideoExporter { let editedAudioBuffer: ArrayBuffer | undefined; let editedAudioMimeType: string | null = null; - if (audioPlan.audioMode === "edited-track") { + if ( + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "offline-render-fallback" + ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { this.reportFinalizingProgress(totalFrames, 99, progress); @@ -741,11 +804,25 @@ export class VideoExporter { audioMode: audioPlan.audioMode, audioSourcePath: audioPlan.audioMode === "copy-source" || - audioPlan.audioMode === "trim-source" + audioPlan.audioMode === "trim-source" || + (audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path") ? audioPlan.audioSourcePath : null, trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + editedTrackSegments: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.editedTrackSegments + : undefined, + audioSourceSampleRate: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.audioSourceSampleRate + : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), @@ -790,7 +867,10 @@ export class VideoExporter { let editedAudioBuffer: ArrayBuffer | undefined; let editedAudioMimeType: string | null = null; - if (audioPlan.audioMode === "edited-track") { + if ( + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "offline-render-fallback" + ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { this.reportFinalizingProgress(totalFrames, 99, progress); @@ -820,11 +900,25 @@ export class VideoExporter { audioMode: audioPlan.audioMode, audioSourcePath: audioPlan.audioMode === "copy-source" || - audioPlan.audioMode === "trim-source" + audioPlan.audioMode === "trim-source" || + (audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path") ? audioPlan.audioSourcePath : null, trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + editedTrackSegments: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.editedTrackSegments + : undefined, + audioSourceSampleRate: + audioPlan.audioMode === "edited-track" && + audioPlan.strategy === "filtergraph-fast-path" + ? audioPlan.audioSourceSampleRate + : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }),