Skip to content

Optimize editor export speed with Rust #1424

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
richiemcilroy wants to merge 2 commits into
base: 0.4.0-bits
Choose a base branch
from
Draft

Optimize editor export speed with Rust #1424

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ababb98
Refactor rendering pipeline and add hardware encoder support
cursoragent Dec 1, 2025
46dbda2
Refactor: Improve H264 encoder selection and configuration
cursoragent Dec 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 123 additions & 39 deletions crates/enc-ffmpeg/src/video/h264.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,53 +267,137 @@ impl H264Encoder {
}
}

fn get_hw_encoder_names() -> &'static [&'static str] {
#[cfg(target_os = "macos")]
{
&["h264_videotoolbox"]
}

#[cfg(target_os = "windows")]
{
&["h264_nvenc", "h264_amf", "h264_qsv", "h264_mf"]
}

#[cfg(target_os = "linux")]
{
&["h264_nvenc", "h264_vaapi", "h264_qsv"]
}

#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
{
&[]
}
}

fn get_codec_and_options(
config: &VideoInfo,
preset: H264Preset,
) -> Option<(Codec, Dictionary<'_>)> {
let encoder_name = {
// if cfg!(target_os = "macos") {
// "libx264"
// // looks terrible rn :(
// // "h264_videotoolbox"
// } else {
// "libx264"
// }

"libx264"
let try_hw_first = matches!(preset, H264Preset::Ultrafast);

let hw_encoders = get_hw_encoder_names();
let sw_encoder = "libx264";

let encoder_names: Vec<&str> = if try_hw_first {
hw_encoders
.iter()
.copied()
.chain(std::iter::once(sw_encoder))
.collect()
} else {
vec![sw_encoder]
};

if let Some(codec) = encoder::find_by_name(encoder_name) {
let mut options = Dictionary::new();

if encoder_name == "h264_videotoolbox" {
options.set("realtime", "true");
} else if encoder_name == "libx264" {
let keyframe_interval_secs = 2;
let keyframe_interval = keyframe_interval_secs * config.frame_rate.numerator();
let keyframe_interval_str = keyframe_interval.to_string();

options.set(
"preset",
match preset {
H264Preset::Slow => "slow",
H264Preset::Medium => "medium",
H264Preset::Ultrafast => "ultrafast",
},
);
if let H264Preset::Ultrafast = preset {
options.set("tune", "zerolatency");
for encoder_name in encoder_names {
if let Some(codec) = encoder::find_by_name(encoder_name) {
let mut options = Dictionary::new();

match encoder_name {
"h264_videotoolbox" => {
options.set("realtime", "false");
options.set("allow_sw", "0");
options.set("prio_speed", "1");
options.set("profile", "high");
options.set("level", "5.1");
debug!("Using VideoToolbox hardware encoder");
return Some((codec, options));
}
"h264_nvenc" => {
options.set("preset", "p1");
options.set("tune", "ll");
options.set("rc", "vbr");
options.set("multipass", "disabled");
options.set("bf", "0");
debug!("Using NVIDIA NVENC hardware encoder");
return Some((codec, options));
}
"h264_amf" => {
options.set("usage", "ultralowlatency");
options.set("quality", "speed");
options.set("rc", "vbr_latency");
options.set("enforce_hrd", "0");
debug!("Using AMD AMF hardware encoder");
return Some((codec, options));
}
"h264_qsv" => {
options.set("preset", "veryfast");
options.set("look_ahead", "0");
options.set("async_depth", "4");
debug!("Using Intel QuickSync hardware encoder");
return Some((codec, options));
}
"h264_vaapi" => {
options.set("rc_mode", "VBR");
options.set("async_depth", "4");
debug!("Using VA-API hardware encoder (Linux)");
return Some((codec, options));
}
"h264_mf" => {
options.set("hw_encoding", "true");
options.set("scenario", "4");
options.set("quality", "1");
debug!("Using MediaFoundation hardware encoder");
return Some((codec, options));
}
"libx264" => {
let keyframe_interval_secs = 2;
let keyframe_interval = keyframe_interval_secs * config.frame_rate.numerator();
let keyframe_interval_str = keyframe_interval.to_string();
let thread_count = thread::available_parallelism()
.map(|v| v.get())
.unwrap_or(4);

options.set(
"preset",
match preset {
H264Preset::Slow => "slow",
H264Preset::Medium => "medium",
H264Preset::Ultrafast => "ultrafast",
},
);
if let H264Preset::Ultrafast = preset {
options.set("tune", "zerolatency");
options.set("bf", "0");
options.set("refs", "1");
options.set("rc-lookahead", "0");
options.set("aq-mode", "0");
options.set("sc_threshold", "0");
}
options.set("vsync", "1");
options.set("g", &keyframe_interval_str);
options.set("keyint_min", &keyframe_interval_str);
options.set("threads", &thread_count.to_string());
options.set("sliced-threads", "1");

debug!(
"Using libx264 software encoder with {} threads",
thread_count
);
return Some((codec, options));
}
_ => continue,
}
options.set("vsync", "1");
options.set("g", &keyframe_interval_str);
options.set("keyint_min", &keyframe_interval_str);
} else if encoder_name == "h264_mf" {
options.set("hw_encoding", "true");
options.set("scenario", "4");
options.set("quality", "1");
}

return Some((codec, options));
}

None
Expand Down
1 change: 0 additions & 1 deletion crates/export/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ use cap_editor::SegmentMedia;
use cap_project::{ProjectConfiguration, RecordingMeta, StudioRecordingMeta};
use cap_rendering::{ProjectRecordingsMeta, RenderVideoConstants};
use std::{path::PathBuf, sync::Arc};
use tracing::error;

#[derive(thiserror::Error, Debug)]
pub enum ExportError {
Expand Down
4 changes: 2 additions & 2 deletions crates/export/src/mp4.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ impl Mp4ExportSettings {
info!("Exporting mp4 with settings: {:?}", &self);
info!("Expected to render {} frames", base.total_frames(self.fps));

let (tx_image_data, mut video_rx) = tokio::sync::mpsc::channel::<(RenderedFrame, u32)>(8);
let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<MP4Input>(8);
let (tx_image_data, mut video_rx) = tokio::sync::mpsc::channel::<(RenderedFrame, u32)>(32);
let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<MP4Input>(16);

let fps = self.fps;

Expand Down
59 changes: 39 additions & 20 deletions crates/rendering/src/decoder/avassetreader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
use crate::DecodedFrame;

use super::frame_converter::{FrameConverter, copy_rgba_plane};
use super::{FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame};
use super::{FRAME_CACHE_SIZE, PREFETCH_LOOKAHEAD, VideoDecoderMessage, pts_to_frame};

#[derive(Clone)]
struct ProcessedFrame {
Expand Down Expand Up @@ -227,6 +227,44 @@

while let Ok(r) = rx.recv() {
match r {
VideoDecoderMessage::PrefetchFrames(start_time_secs, prefetch_fps) => {
let start_frame = (start_time_secs * prefetch_fps as f32).floor() as u32;
let end_frame = start_frame + PREFETCH_LOOKAHEAD as u32;

for frame in &mut frames {
let Ok(frame) = frame else { continue };

let current_frame = pts_to_frame(
frame.pts().value,
Rational::new(1, frame.pts().scale),
fps,
);

if let Some(image_buf) = frame.image_buf() {
if current_frame >= start_frame
&& current_frame <= end_frame
&& !cache.contains_key(&current_frame)
{
if cache.len() >= FRAME_CACHE_SIZE {
if let Some(&oldest) = cache.keys().next() {
cache.remove(&oldest);
}
}

Check failure on line 252 in crates/rendering/src/decoder/avassetreader.rs

View workflow job for this annotation

GitHub Actions / Clippy (aarch64-apple-darwin, macos-latest)

this `if` statement can be collapsed 

error: this `if` statement can be collapsed
   --> crates/rendering/src/decoder/avassetreader.rs:248:33
    |
248 | / ...                   if cache.len() >= FRAME_CACHE_SIZE {
249 | | ...                       if let Some(&oldest) = cache.keys().next() {
250 | | ...                           cache.remove(&oldest);
251 | | ...                       }
252 | | ...                   }
    | |_______________________^
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.91.0/index.html#collapsible_if
help: collapse nested if block
    |
248 ~                                 if cache.len() >= FRAME_CACHE_SIZE
249 ~                                     && let Some(&oldest) = cache.keys().next() {
250 |                                         cache.remove(&oldest);
251 ~                                     }
    |
cache.insert(
current_frame,
CachedFrame::Raw {
image_buf: image_buf.retained(),
number: current_frame,
},
);
}
}

Check failure on line 261 in crates/rendering/src/decoder/avassetreader.rs

View workflow job for this annotation

GitHub Actions / Clippy (aarch64-apple-darwin, macos-latest)

this `if` statement can be collapsed 

error: this `if` statement can be collapsed
   --> crates/rendering/src/decoder/avassetreader.rs:243:25
    |
243 | /                         if let Some(image_buf) = frame.image_buf() {
244 | |                             if current_frame >= start_frame
245 | |                                 && current_frame <= end_frame
246 | |                                 && !cache.contains_key(&current_frame)
...   |
261 | |                         }
    | |_________________________^
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.91.0/index.html#collapsible_if
    = note: requested on the command line with `-D clippy::collapsible-if`
help: collapse nested if block
    |
243 ~                         if let Some(image_buf) = frame.image_buf()
244 ~                             && current_frame >= start_frame
245 |                                 && current_frame <= end_frame
...
259 |                                 );
260 ~                             }
    |

if current_frame >= end_frame {
break;
}
}
}
VideoDecoderMessage::GetFrame(requested_time, sender) => {
let requested_frame = (requested_time * fps as f32).floor() as u32;

Expand Down Expand Up @@ -297,8 +335,6 @@

this.is_done = false;

// Handles frame skips.
// We use the cache instead of last_sent_frame as newer non-matching frames could have been decoded.
if let Some(most_recent_prev_frame) =
cache.iter_mut().rev().find(|v| *v.0 < requested_frame)
&& let Some(sender) = sender.take()
Expand All @@ -314,7 +350,6 @@
&& let Some(sender) = sender.take()
{
let data = cache_frame.process(&mut processor);
// info!("sending frame {requested_frame}");

(sender)(data);

Expand Down Expand Up @@ -344,23 +379,13 @@
}

if current_frame > requested_frame && sender.is_some() {
// not inlining this is important so that last_sent_frame is dropped before the sender is invoked
let last_sent_frame = last_sent_frame.borrow().clone();

if let Some((sender, last_sent_frame)) =
last_sent_frame.and_then(|l| Some((sender.take()?, l)))
{
// info!(
// "sending previous frame {} for {requested_frame}",
// last_sent_frame.0
// );

(sender)(last_sent_frame);
} else if let Some(sender) = sender.take() {
// info!(
// "sending forward frame {current_frame} for {requested_frame}",
// );

(sender)(cache_frame.process(&mut processor));
}
}
Expand All @@ -374,14 +399,8 @@

this.is_done = true;

// not inlining this is important so that last_sent_frame is dropped before the sender is invoked
let last_sent_frame = last_sent_frame.borrow().clone();
if let Some((sender, last_sent_frame)) = sender.take().zip(last_sent_frame) {
// info!(
// "sending hail mary frame {} for {requested_frame}",
// last_sent_frame.0
// );

(sender)(last_sent_frame);
}
}
Expand Down
Loading
Loading