diff --git a/Cargo.lock b/Cargo.lock index d97726607d6..6f739045773 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1140,7 +1140,7 @@ dependencies = [ [[package]] name = "cap-desktop" -version = "0.3.71" +version = "0.3.72" dependencies = [ "anyhow", "axum", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index a7613c2399e..2b53cdb3203 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cap-desktop" -version = "0.3.71" +version = "0.3.72" description = "Beautiful screen recordings, owned by you." authors = ["you"] edition = "2024" diff --git a/crates/enc-mediafoundation/src/mft.rs b/crates/enc-mediafoundation/src/mft.rs index 5a0babf7bab..808a5cab67d 100644 --- a/crates/enc-mediafoundation/src/mft.rs +++ b/crates/enc-mediafoundation/src/mft.rs @@ -21,16 +21,29 @@ pub struct EncoderDevice { impl EncoderDevice { pub fn enumerate(major_type: GUID, subtype: GUID) -> Result> { + Self::enumerate_with_flags( + major_type, + subtype, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY | MFT_ENUM_FLAG_SORTANDFILTER, + ) + } + + pub fn enumerate_with_flags( + major_type: GUID, + subtype: GUID, + flags: MFT_ENUM_FLAG, + ) -> Result> { let output_info = MFT_REGISTER_TYPE_INFO { guidMajorType: major_type, guidSubtype: subtype, }; - let encoders = enumerate_mfts( - &MFT_CATEGORY_VIDEO_ENCODER, - MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY | MFT_ENUM_FLAG_SORTANDFILTER, - None, - Some(&output_info), - )?; + let flags = if flags.0 == 0 { + MFT_ENUM_FLAG_SORTANDFILTER + } else { + flags | MFT_ENUM_FLAG_SORTANDFILTER + }; + let encoders = + enumerate_mfts(&MFT_CATEGORY_VIDEO_ENCODER, flags, None, Some(&output_info))?; let mut encoder_devices = Vec::new(); for encoder in encoders { let display_name = if let Some(display_name) = diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index ca96a5a1f6e..493634853d4 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -15,7 +15,8 @@ use windows::{ MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, MFCreateDXGIDeviceManager, MFCreateDXGISurfaceBuffer, MFCreateMediaType, - MFCreateSample, MFMediaType_Video, MFT_MESSAGE_COMMAND_FLUSH, + MFCreateSample, MFMediaType_Video, MFT_ENUM_FLAG, MFT_ENUM_FLAG_HARDWARE, + MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_OF_STREAM, MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_SET_TYPE_TEST_ONLY, @@ -97,13 +98,15 @@ pub enum HandleNeedsInputError { unsafe impl Send for H264Encoder {} impl H264Encoder { - pub fn new_with_scaled_output( + fn new_with_scaled_output_with_flags( d3d_device: &ID3D11Device, format: DXGI_FORMAT, input_resolution: SizeInt32, output_resolution: SizeInt32, frame_rate: u32, bitrate_multipler: f32, + flags: MFT_ENUM_FLAG, + enable_hardware_transforms: bool, ) -> Result { let bitrate = calculate_bitrate( output_resolution.Width as u32, @@ -112,13 +115,14 @@ impl H264Encoder { bitrate_multipler, ); - let transform = EncoderDevice::enumerate(MFMediaType_Video, MFVideoFormat_H264) - .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)? - .first() - .cloned() - .ok_or(NewVideoEncoderError::NoVideoEncoderDevice)? - .create_transform() - .map_err(NewVideoEncoderError::EncoderTransform)?; + let transform = + EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags) + .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)? + .first() + .cloned() + .ok_or(NewVideoEncoderError::NoVideoEncoderDevice)? + .create_transform() + .map_err(NewVideoEncoderError::EncoderTransform)?; let video_processor = VideoProcessor::new( d3d_device.clone(), @@ -130,7 +134,6 @@ impl H264Encoder { ) .map_err(NewVideoEncoderError::VideoProcessor)?; - // Create MF device manager let mut device_manager_reset_token: u32 = 0; let media_device_manager = { let mut media_device_manager = None; @@ -149,7 +152,6 @@ impl H264Encoder { .map_err(NewVideoEncoderError::DeviceManager)? }; - // Setup MFTransform let event_generator: IMFMediaEventGenerator = transform .cast() .map_err(NewVideoEncoderError::EventGenerator)?; @@ -163,7 +165,10 @@ impl H264Encoder { .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) .map_err(NewVideoEncoderError::EventGenerator)?; attributes - .SetUINT32(&MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1) + .SetUINT32( + &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, + enable_hardware_transforms as u32, + ) .map_err(NewVideoEncoderError::EventGenerator)?; }; @@ -182,13 +187,6 @@ impl H264Encoder { match result { Ok(_) => {} Err(error) => { - // https://docs.microsoft.com/en-us/windows/win32/api/mftransform/nf-mftransform-imftransform-getstreamids - // This method can return E_NOTIMPL if both of the following conditions are true: - // * The transform has a fixed number of streams. - // * The streams are numbered consecutively from 0 to n – 1, where n is the - // number of input streams or output streams. In other words, the first - // input stream is 0, the second is 1, and so on; and the first output - // stream is 0, the second is 1, and so on. if error.code() == E_NOTIMPL { for i in 0..number_of_input_streams { input_stream_ids[i as usize] = i; @@ -206,7 +204,6 @@ impl H264Encoder { let input_stream_id = input_stream_ids[0]; let output_stream_id = output_stream_ids[0]; - // TOOD: Avoid this AddRef? unsafe { let temp = media_device_manager.clone(); transform @@ -300,6 +297,46 @@ impl H264Encoder { }) } + pub fn new_with_scaled_output( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_with_flags( + d3d_device, + format, + input_resolution, + output_resolution, + frame_rate, + bitrate_multipler, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY, + true, + ) + } + + pub fn new_with_scaled_output_software( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_with_flags( + d3d_device, + format, + input_resolution, + output_resolution, + frame_rate, + bitrate_multipler, + MFT_ENUM_FLAG_TRANSCODE_ONLY, + false, + ) + } + pub fn new( d3d_device: &ID3D11Device, format: DXGI_FORMAT, @@ -317,6 +354,23 @@ impl H264Encoder { ) } + pub fn new_software( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_software( + d3d_device, + format, + resolution, + resolution, + frame_rate, + bitrate_multipler, + ) + } + pub fn bitrate(&self) -> u32 { self.bitrate } diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index 10ee75f1623..cc029eacff8 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -246,20 +246,25 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { cap_mediafoundation_utils::thread_init(); let screen_config = source.0.info(); + let frame_rate = source.0.config().fps(); + let bitrate_multiplier = 0.1f32; + let d3d_device = source.0.d3d_device().clone(); + let pixel_format = screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(); + let capture_resolution = SizeInt32 { + Width: screen_config.width as i32, + Height: screen_config.height as i32, + }; let mut output = ffmpeg::format::output(&output_path) .map_err(|e| MediaError::Any(format!("CreateOutput: {e}").into()))?; let screen_encoder = { let native_encoder = cap_enc_mediafoundation::H264Encoder::new( - source.0.d3d_device(), - screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(), - SizeInt32 { - Width: screen_config.width as i32, - Height: screen_config.height as i32, - }, - source.0.config().fps(), - 0.1, + &d3d_device, + pixel_format, + capture_resolution, + frame_rate, + bitrate_multiplier, ); match native_encoder { @@ -312,26 +317,92 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { let _ = ready.send(Ok(())); let mut timestamp_tx = Some(timestamp_tx); - - while let Ok(e) = encoder.get_event() { + let mut pending_frame: Option<( + Self::VideoFormat, + Timestamp, + windows::Foundation::TimeSpan, + )> = None; + let mut using_software_encoder = false; + + 'event_loop: while let Ok(e) = encoder.get_event() { match e { MediaFoundation::METransformNeedInput => { - let Ok((frame, timestamp)) = source.1.recv() else { - break; + let (mut frame, timestamp, frame_time) = if let Some(pending) = + pending_frame.take() + { + pending + } else { + let Ok((frame, timestamp)) = source.1.recv() else { + break; + }; + + if let Some(timestamp_tx) = timestamp_tx.take() { + let _ = timestamp_tx.send(timestamp); + } + + let frame_time = frame + .inner() + .SystemRelativeTime() + .map_err(|e| format!("FrameTime: {e}"))?; + + (frame, timestamp, frame_time) }; - if let Some(timestamp_tx) = timestamp_tx.take() { - timestamp_tx.send(timestamp).unwrap(); + loop { + match encoder.handle_needs_input( + frame.texture(), + frame_time, + ) { + Ok(()) => break, + Err( + cap_enc_mediafoundation::video::HandleNeedsInputError::ProcessInput( + error, + ), + ) => { + use tracing::warn; + use windows::Win32::Foundation::E_FAIL; + + if !using_software_encoder && error.code() == E_FAIL { + warn!( + "Native H264 ProcessInput failed with {:?}; falling back to software encoder", + error.code() + ); + pending_frame = + Some((frame, timestamp, frame_time)); + + let mut software_encoder = + cap_enc_mediafoundation::H264Encoder::new_software( + &d3d_device, + pixel_format, + capture_resolution, + frame_rate, + bitrate_multiplier, + ) + .map_err(|e| format!( + "SoftwareEncoderInit: {e}" + ))?; + software_encoder + .start() + .map_err(|e| { + format!( + "ScreenEncoderStart: {e}" + ) + })?; + + encoder = software_encoder; + using_software_encoder = true; + continue 'event_loop; + } + + return Err(format!( + "NeedsInput: ProcessInput: {error}" + )); + } + Err(err) => { + return Err(format!("NeedsInput: {err}")); + } + } } - - let frame_time = frame - .inner() - .SystemRelativeTime() - .map_err(|e| format!("FrameTime: {e}"))?; - - encoder - .handle_needs_input(frame.texture(), frame_time) - .map_err(|e| format!("NeedsInput: {e}"))?; } MediaFoundation::METransformHaveOutput => { if let Some(output_sample) = encoder @@ -427,24 +498,30 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { let has_audio_sources = audio_mixer.has_sources(); let screen_config = source.0.info(); + let frame_rate = 30u32; + let bitrate_multiplier = 0.15f32; + let d3d_device = source.0.d3d_device().clone(); + let pixel_format = screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(); + let input_resolution = SizeInt32 { + Width: screen_config.width as i32, + Height: screen_config.height as i32, + }; + let output_resolution = SizeInt32 { + Width: screen_config.width as i32, + Height: screen_config.height as i32, + }; let mut output = ffmpeg::format::output(&output_path) .map_err(|e| MediaError::Any(format!("CreateOutput: {e}").into()))?; let screen_encoder = { let native_encoder = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - source.0.d3d_device(), - screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(), - SizeInt32 { - Width: screen_config.width as i32, - Height: screen_config.height as i32, - }, - SizeInt32 { - Width: screen_config.width as i32, - Height: screen_config.height as i32, - }, - 30, - 0.15, + &d3d_device, + pixel_format, + input_resolution, + output_resolution, + frame_rate, + bitrate_multiplier, ); match native_encoder { @@ -454,7 +531,7 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { cap_mediafoundation_ffmpeg::MuxerConfig { width: screen_config.width, height: screen_config.height, - fps: 30, + fps: frame_rate, bitrate: screen_encoder.bitrate(), }, ) @@ -539,21 +616,36 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { let _ = ready.send(Ok(())); let mut first_frame_tx = Some(first_frame_tx); + let mut pending_frame: Option<( + Self::VideoFormat, + windows::Foundation::TimeSpan, + )> = None; + let mut using_software_encoder = false; - while let Ok(e) = encoder.get_event() { + 'event_loop: while let Ok(e) = encoder.get_event() { match e { MediaFoundation::METransformNeedInput => { use cap_timestamp::PerformanceCounterTimestamp; + use tracing::warn; + use windows::Win32::Foundation::E_FAIL; - let Ok((frame, _)) = source.1.recv() else { - break; + let (mut frame, frame_time) = if let Some(pending) = + pending_frame.take() + { + pending + } else { + let Ok((frame, _)) = source.1.recv() else { + break; + }; + + let frame_time = frame + .inner() + .SystemRelativeTime() + .map_err(|e| format!("Frame Time: {e}"))?; + + (frame, frame_time) }; - let frame_time = frame - .inner() - .SystemRelativeTime() - .map_err(|e| format!("Frame Time: {e}"))?; - let timestamp = Timestamp::PerformanceCounter( PerformanceCounterTimestamp::new(frame_time.Duration), ); @@ -562,9 +654,56 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { let _ = first_frame_tx.send(timestamp); } - encoder - .handle_needs_input(frame.texture(), frame_time) - .map_err(|e| format!("NeedsInput: {e}"))?; + loop { + match encoder.handle_needs_input( + frame.texture(), + frame_time, + ) { + Ok(()) => break, + Err( + cap_enc_mediafoundation::video::HandleNeedsInputError::ProcessInput( + error, + ), + ) => { + if !using_software_encoder && error.code() == E_FAIL { + warn!( + "Native H264 ProcessInput failed with {:?}; falling back to software encoder", + error.code() + ); + pending_frame = Some((frame, frame_time)); + + let mut software_encoder = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output_software( + &d3d_device, + pixel_format, + input_resolution, + output_resolution, + frame_rate, + bitrate_multiplier, + ) + .map_err(|e| { + format!("SoftwareEncoderInit: {e}") + })?; + + software_encoder + .start() + .map_err(|e| format!( + "StartScreenEncoder: {e}" + ))?; + + encoder = software_encoder; + using_software_encoder = true; + continue 'event_loop; + } + + return Err(format!( + "NeedsInput: ProcessInput: {error}" + )); + } + Err(err) => { + return Err(format!("NeedsInput: {err}")); + } + } + } } MediaFoundation::METransformHaveOutput => { if let Some(output_sample) = encoder