diff --git a/Cargo.lock b/Cargo.lock index a63ba98e6..10c4f59a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,55 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.86" @@ -162,7 +211,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -307,7 +356,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -342,7 +391,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -508,7 +557,7 @@ dependencies = [ "bitflags 1.3.2", "cexpr 0.4.0", "clang-sys", - "clap", + "clap 2.34.0", "env_logger", "lazy_static", "lazycell", @@ -540,7 +589,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex 1.3.0", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -558,7 +607,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex 1.3.0", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -690,7 +739,7 @@ checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -800,6 +849,7 @@ dependencies = [ "cap-flags", "cap-gpu-converters", "cap-project", + "cidre", "cocoa 0.26.0", "core-foundation 0.10.0", "core-graphics 0.24.0", @@ -818,6 +868,7 @@ dependencies = [ "objc2-foundation", "ringbuf", "scap", + "screencapturekit", "serde", "specta", "tempfile", @@ -996,6 +1047,21 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "cidre" +version = "0.4.0" +source = "git+https://github.com/yury/cidre?rev=1e008bec49a0f97aeaaea6130a0ba20fe00aa03b#1e008bec49a0f97aeaaea6130a0ba20fe00aa03b" +dependencies = [ + "cidre-macros", + "parking_lot", + "tokio", +] + +[[package]] +name = "cidre-macros" +version = "0.1.0" +source = "git+https://github.com/yury/cidre?rev=1e008bec49a0f97aeaaea6130a0ba20fe00aa03b#1e008bec49a0f97aeaaea6130a0ba20fe00aa03b" + [[package]] name = "clang-sys" version = "1.8.1" @@ -1022,6 +1088,46 @@ dependencies = [ "vec_map", ] +[[package]] +name = "clap" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + [[package]] name = "claxon" version = "0.4.3" @@ -1119,6 +1225,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "com" version = "0.6.0" @@ -1518,7 +1630,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1528,7 +1640,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", +] + +[[package]] +name = "ctrlc" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" +dependencies = [ + "nix 0.29.0", + "windows-sys 0.59.0", ] [[package]] @@ -1563,7 +1685,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1574,7 +1696,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1635,7 +1757,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1648,7 +1770,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1717,7 +1839,6 @@ dependencies = [ "uuid", "wgpu", "windows 0.52.0", - "windows-capture", ] [[package]] @@ -1801,7 +1922,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1833,7 +1954,7 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1986,7 +2107,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2249,7 +2370,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2373,7 +2494,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2651,7 +2772,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2820,7 +2941,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3248,7 +3369,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3276,6 +3397,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -4193,7 +4320,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4254,7 +4381,7 @@ dependencies = [ "proc-macro-crate 2.0.2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4596,7 +4723,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4846,7 +4973,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4893,7 +5020,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5102,7 +5229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5687,8 +5814,8 @@ dependencies = [ [[package]] name = "scap" -version = "0.0.6" -source = "git+https://github.com/CapSoftware/scap?rev=4d6be030ba2b0cea565ccaee28b1999f25b8dd5d#4d6be030ba2b0cea565ccaee28b1999f25b8dd5d" +version = "0.0.7" +source = "git+https://github.com/CapSoftware/scap?rev=b1e140a3fe90#b1e140a3fe905c19b845dfea66b3b1aea02f0472" dependencies = [ "cocoa 0.25.0", "core-graphics-helmer-fork", @@ -5700,7 +5827,7 @@ dependencies = [ "screencapturekit-sys", "sysinfo", "tao-core-video-sys", - "windows 0.52.0", + "windows 0.58.0", "windows-capture", ] @@ -5737,7 +5864,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5855,7 +5982,7 @@ checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5866,7 +5993,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5899,7 +6026,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5950,7 +6077,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -6162,7 +6289,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -6333,9 +6460,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -6474,7 +6601,7 @@ checksum = "f4e16beb8b2ac17db28eab8bca40e62dbfbb34c0fcdc6d9826b11b7b5d047dfd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -6587,7 +6714,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "syn 2.0.85", + "syn 2.0.87", "tauri-utils", "thiserror", "time", @@ -6605,7 +6732,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "tauri-codegen", "tauri-utils", ] @@ -6934,7 +7061,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -7044,22 +7171,22 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -7145,7 +7272,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -7316,7 +7443,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -7534,6 +7661,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.10.0" @@ -7672,7 +7805,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -7706,7 +7839,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7875,7 +8008,7 @@ checksum = "1d228f15bba3b9d56dde8bddbee66fa24545bd17b48d5128ccf4a8742b18e431" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8136,14 +8269,16 @@ dependencies = [ [[package]] name = "windows-capture" -version = "1.2.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e2f94b43842205eb84814f505badade792e7b8cd132e436636ccbf7baa08fc" +checksum = "308113a004a94ea5fb83dc7d19e2c66050879b6767f1eb10231af2e77b66475d" dependencies = [ + "clap 4.5.20", + "ctrlc", "parking_lot", "rayon", "thiserror", - "windows 0.56.0", + "windows 0.58.0", ] [[package]] @@ -8207,7 +8342,7 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8218,7 +8353,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8229,7 +8364,7 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8240,7 +8375,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8705,7 +8840,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 580dd7d74..0bf66bc19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ tokio = { version = "1.39.3", features = [ ] } tauri = { version = "2.0.0" } specta = { version = "=2.0.0-rc.20" } -scap = { git = "https://github.com/CapSoftware/scap", rev = "4d6be030ba2b0cea565ccaee28b1999f25b8dd5d" } +scap = { git = "https://github.com/CapSoftware/scap", rev = "b1e140a3fe90" } nokhwa = { git = "https://github.com/CapSoftware/nokhwa", rev = "c5c7e2298764", features = [ "input-native", "serialize", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 5be87f7c5..31b11e587 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -96,9 +96,6 @@ windows = { version = "0.52.0", features = [ "Win32_UI_WindowsAndMessaging", "Win32_Graphics_Gdi", ] } -# Lock the version of windows-capture used by scap@691bd88798d3 -# TODO: Remove this once scap uses the latest version of `windows` and `windows-capture` -windows-capture = "=1.2.0" [target.'cfg(unix)'.dependencies] nix = { version = "0.29.0", features = ["fs"] } diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 24c81747d..9076fa790 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -21,7 +21,7 @@ use cap_editor::{EditorInstance, FRAMES_WS_PATH}; use cap_editor::{EditorState, ProjectRecordings}; use cap_media::sources::CaptureScreen; use cap_media::{ - feeds::{AudioData, AudioFrameBuffer, CameraFeed, CameraFrameSender}, + feeds::{AudioFrameBuffer, CameraFeed, CameraFrameSender}, platform::Bounds, sources::{AudioInputSource, ScreenCaptureTarget}, }; @@ -34,10 +34,10 @@ use cap_rendering::{ProjectUniforms, ZOOM_DURATION}; use general_settings::GeneralSettingsStore; use image::{ImageBuffer, Rgba}; use mp4::Mp4Reader; -use num_traits::ToBytes; use png::{ColorType, Encoder}; use recording::{ - list_cameras, list_capture_screens, list_capture_windows, InProgressRecording, FPS, + list_cameras, list_capture_screens, list_capture_windows, InProgressRecording, + RecordingSettingsStore, }; use scap::capturer::Capturer; use scap::frame::Frame; @@ -338,10 +338,13 @@ async fn start_recording(app: AppHandle, state: MutableState<'_, App>) -> Result } } + let recording_settings = RecordingSettingsStore::get(&app)?; + match recording::start( id, recording_dir, &state.start_recording_options, + recording_settings, state.camera_feed.as_ref(), ) .await @@ -969,6 +972,7 @@ async fn render_to_file_impl( let audio = editor_instance.audio.clone(); let decoders = editor_instance.decoders.clone(); let options = editor_instance.render_constants.options.clone(); + let screen_fps = editor_instance.recordings.display.fps; let (tx_image_data, mut rx_image_data) = tokio::sync::mpsc::unbounded_channel::>(); @@ -997,7 +1001,7 @@ async fn render_to_file_impl( ffmpeg.add_input(cap_ffmpeg_cli::FFmpegRawVideoInput { width: output_size.0, height: output_size.1, - fps: 30, + fps: screen_fps, pix_fmt: "rgba", input: pipe_path.clone().into_os_string(), }); @@ -1082,7 +1086,7 @@ async fn render_to_file_impl( let audio_info = audio.buffer.info(); let estimated_samples_per_frame = - f64::from(audio_info.sample_rate) / f64::from(FPS); + f64::from(audio_info.sample_rate) / f64::from(screen_fps); let samples = estimated_samples_per_frame.ceil() as usize; if let Some((_, frame_data)) = @@ -1158,6 +1162,7 @@ async fn render_to_file_impl( decoders, editor_instance.cursor.clone(), editor_instance.project_path.clone(), + screen_fps, ) .await?; @@ -1629,10 +1634,9 @@ async fn render_to_file( .await .unwrap(); - // 30 FPS (calculated for output video) - let total_frames = (duration * 30.0).round() as u32; - let editor_instance = upsert_editor_instance(&app, video_id.clone()).await; + let screen_fps = editor_instance.recordings.display.fps as f64; + let total_frames = (duration * screen_fps).round() as u32; render_to_file_impl( &editor_instance, @@ -2459,6 +2463,7 @@ pub async fn run() { check_upgraded_and_update, open_external_link, hotkeys::set_hotkey, + recording::set_recording_settings, set_general_settings, delete_auth_open_signin, reset_camera_permissions, @@ -2489,6 +2494,7 @@ pub async fn run() { .typ::() .typ::() .typ::() + .typ::() .typ::(); #[cfg(debug_assertions)] @@ -2528,6 +2534,7 @@ pub async fn run() { specta_builder.mount_events(app); hotkeys::init(app.handle()); general_settings::init(app.handle()); + recording::init_settings(app.handle()); let app_handle = app.handle().clone(); diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index cbad0e821..ade2e1727 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -1,32 +1,95 @@ use cap_flags::FLAGS; use cap_media::{encoders::*, feeds::*, filters::*, pipeline::*, sources::*, MediaError}; -use cap_project::{CursorClickEvent, CursorMoveEvent, RecordingMeta}; -use serde::Serialize; +use cap_project::{CursorClickEvent, CursorMoveEvent, RecordingMeta, TargetFPS, TargetResolution}; +use serde::{Deserialize, Serialize}; use specta::Type; use std::collections::HashMap; use std::fs::File; +use std::path::PathBuf; use std::sync::atomic::AtomicBool; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; -use std::path::PathBuf; +use tauri::{AppHandle, Manager, Wry}; +use tauri_plugin_store::StoreExt; use tokio::sync::oneshot; use crate::cursor::spawn_cursor_recorder; use crate::RecordingOptions; -// TODO: Hacky, please fix -pub const FPS: u32 = 30; +#[derive(Serialize, Deserialize, Type, Debug)] +pub struct RecordingSettingsStore { + pub use_hardware_acceleration: bool, + #[serde(default = "default_recording_resolution")] + pub capture_resolution: Option, + #[serde(default = "default_recording_resolution")] + pub output_resolution: Option, + pub recording_fps: TargetFPS, +} + +fn default_recording_resolution() -> Option { + Some(TargetResolution::_1080p) +} + +impl Default for RecordingSettingsStore { + fn default() -> Self { + Self { + use_hardware_acceleration: false, + capture_resolution: Some(TargetResolution::_1080p), + output_resolution: None, + recording_fps: TargetFPS::_30, + } + } +} + +pub type RecordingSettingsState = Mutex; + +pub fn init_settings(app: &AppHandle) { + println!("Initializing RecordingSettingsStore"); + let store = RecordingSettingsStore::get(app) + .unwrap() + .unwrap_or_default(); + app.manage(RecordingSettingsState::new(store)); + println!("RecordingSettingsState managed"); +} + +impl RecordingSettingsStore { + pub fn get(app: &AppHandle) -> Result, String> { + let Some(Some(store)) = app.get_store("store").map(|s| s.get("recording_settings")) else { + return Ok(None); + }; + + serde_json::from_value(store).map_err(|e| e.to_string()) + } + + pub fn set(app: &AppHandle, settings: Self) -> Result<(), String> { + let Some(store) = app.get_store("store") else { + return Err("Store not found".to_string()); + }; + + store.set("recording_settings", serde_json::json!(settings)); + store.save().map_err(|e| e.to_string()) + } +} + +#[tauri::command(async)] +#[specta::specta] +pub fn set_recording_settings( + app: AppHandle, + settings: RecordingSettingsStore, +) -> Result<(), String> { + RecordingSettingsStore::set(&app, settings) +} #[tauri::command(async)] #[specta::specta] pub fn list_capture_screens() -> Vec { - ScreenCaptureSource::list_screens() + ScreenCaptureSource::::list_screens() } #[tauri::command(async)] #[specta::specta] pub fn list_capture_windows() -> Vec { - ScreenCaptureSource::list_targets() + ScreenCaptureSource::::list_targets() } #[tauri::command(async)] @@ -186,6 +249,7 @@ pub async fn start( id: String, recording_dir: PathBuf, recording_options: &RecordingOptions, + recording_settings: Option, camera_feed: Option<&CameraFeed>, ) -> Result { let content_dir = recording_dir.join("content"); @@ -201,22 +265,53 @@ pub async fn start( let mut audio_output_path = None; let mut camera_output_path = None; - let screen_source = - ScreenCaptureSource::init(dbg!(&recording_options.capture_target), None, None); - let screen_config = screen_source.info(); - let screen_bounds = screen_source.bounds; - - let output_config = screen_config.scaled(1920, 30); - let screen_filter = VideoFilter::init("screen", screen_config, output_config)?; - let screen_encoder = H264Encoder::init( - "screen", - output_config, - Output::File(display_output_path.clone()), - )?; - pipeline_builder = pipeline_builder - .source("screen_capture", screen_source) - .pipe("screen_capture_filter", screen_filter) - .sink("screen_capture_encoder", screen_encoder); + let settings = recording_settings.unwrap_or_default(); + + if settings.use_hardware_acceleration && cfg!(target_os = "macos") { + let screen_source = ScreenCaptureSource::::init( + dbg!(&recording_options.capture_target), + settings.recording_fps, + settings.capture_resolution, + ); + let screen_config = screen_source.info(); + let output_config = settings.output_resolution.map(|output| { + screen_config + .with_resolution(output.to_width()) + .with_hardware_format() + }); + + let screen_encoder = cap_media::encoders::H264AVAssetWriterEncoder::init( + "screen", + output_config, + Output::File(display_output_path.clone()), + )?; + pipeline_builder = pipeline_builder + .source("screen_capture", screen_source) + .sink("screen_capture_encoder", screen_encoder); + } else { + let screen_source = ScreenCaptureSource::::init( + dbg!(&recording_options.capture_target), + settings.recording_fps, + None, + ); + let screen_config = screen_source.info(); + + let mut output_config = screen_config.with_software_format(); + if let Some(output_resolution) = settings.output_resolution { + output_config = output_config.with_resolution(output_resolution.to_width()); + } + let screen_filter = VideoFilter::init("screen", screen_config, output_config)?; + + let screen_encoder = H264Encoder::init( + "screen", + output_config, + Output::File(display_output_path.clone()), + )?; + pipeline_builder = pipeline_builder + .source("screen_capture", screen_source) + .pipe("screen_capture_filter", screen_filter) + .sink("screen_capture_encoder", screen_encoder); + } if let Some(mic_source) = recording_options .audio_input_name @@ -226,7 +321,6 @@ pub async fn start( let mic_config = mic_source.info(); audio_output_path = Some(content_dir.join("audio-input.mp3")); - // let mic_filter = AudioFilter::init("microphone", mic_config, "aresample=async=1:min_hard_comp=0.100000:first_pts=0")?; let mic_encoder = MP3Encoder::init( "microphone", mic_config, @@ -235,13 +329,18 @@ pub async fn start( pipeline_builder = pipeline_builder .source("microphone_capture", mic_source) - // .pipe("microphone_filter", mic_filter) .sink("microphone_encoder", mic_encoder); } if let Some(camera_source) = CameraSource::init(camera_feed) { let camera_config = camera_source.info(); - let output_config = camera_config.scaled(1920, 30); + // TODO: I'm not sure if there's a point to scaling the camera capture to the same resolution + // as the display capture (since it will be scaled down anyway, but matching at least the frame + // rate here is easier than trying to sync different frame rates while editing/rendering). + // Also, use hardware filters maybe? + let output_config = camera_config + .with_software_format() + .with_fps(settings.recording_fps.to_raw()); camera_output_path = Some(content_dir.join("camera.mp4")); let camera_filter = VideoFilter::init("camera", camera_config, output_config)?; @@ -263,9 +362,11 @@ pub async fn start( let stop_signal = Arc::new(AtomicBool::new(false)); // Initialize default values for cursor channels - let (mouse_moves, mouse_clicks) = if FLAGS.record_mouse { - spawn_cursor_recorder(stop_signal.clone(), screen_bounds, content_dir, cursors_dir) - } else { + let (mouse_moves, mouse_clicks) = + // if FLAGS.record_mouse + { + // spawn_cursor_recorder(stop_signal.clone(), screen_bounds, content_dir, cursors_dir) + // } else { // Create dummy channels that will never receive data let (move_tx, move_rx) = oneshot::channel(); let (click_tx, click_rx) = oneshot::channel(); diff --git a/apps/desktop/src/components/SwitchButton.tsx b/apps/desktop/src/components/SwitchButton.tsx new file mode 100644 index 000000000..39898f684 --- /dev/null +++ b/apps/desktop/src/components/SwitchButton.tsx @@ -0,0 +1,34 @@ +export type SwitchButtonProps = { + name: T, + value: boolean, + disabled?: boolean, + onChange: (name: T, value: boolean) => void, +}; + +export function SwitchButton(props: SwitchButtonProps) { + return ( + + ); +} diff --git a/apps/desktop/src/routes/(window-chrome)/settings.tsx b/apps/desktop/src/routes/(window-chrome)/settings.tsx index 410070708..f993b95b4 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings.tsx @@ -24,6 +24,7 @@ export default function (props: RouteSectionProps) { each={[ { href: "general", name: "General", icon: IconCapSettings }, { href: "hotkeys", name: "Shortcuts", icon: IconCapHotkeys }, + { href: "recording", name: "Recording", icon: IconLucideVideo }, { href: "recordings", name: "Previous Recordings", @@ -36,25 +37,19 @@ export default function (props: RouteSectionProps) { }, ]} > - {(item) => { - const isActive = () => location.pathname.includes(item.href); - return ( -
  • - - - {item.name} - -
  • - ); - }} + {(item) => ( +
  • + + + {item.name} + +
  • + )}
    diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index 2b5767a83..8e60ca265 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -6,8 +6,16 @@ import { isPermissionGranted, requestPermission, } from "@tauri-apps/plugin-notification"; +import { SwitchButton } from "~/components/SwitchButton"; -const settingsList = [ +type Setting = { + key: keyof GeneralSettingsStore, + label: string, + description: string, + requiresPermission?: boolean, +}; + +const settingsList: Setting[] = [ { key: "upload_individual_files", label: "Upload individual recording files when creating shareable link", @@ -62,7 +70,7 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) { } ); - const handleChange = async (key: string, value: boolean) => { + const handleChange = async (key: keyof GeneralSettingsStore, value: boolean) => { console.log(`Handling settings change for ${key}: ${value}`); // Special handling for notifications permission if (key === "enable_notifications") { @@ -86,7 +94,7 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) { } } - setSettings(key as keyof GeneralSettingsStore, value); + setSettings(key, value); await commands.setGeneralSettings({ ...settings, [key]: value, @@ -102,47 +110,11 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) {

    {setting.label}

    - +
    {setting.description && (

    {setting.description}

    diff --git a/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx b/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx new file mode 100644 index 000000000..72a966d72 --- /dev/null +++ b/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx @@ -0,0 +1,206 @@ +import { createResource, Show, For } from "solid-js"; +import { createStore } from "solid-js/store"; +import { Select as KSelect } from "@kobalte/core/select"; +import { recordingSettingsStore } from "~/store"; +import { createCurrentRecordingQuery } from "~/utils/queries"; +import { + commands, + type RecordingSettingsStore, + TargetResolution, + TargetFPS, +} from "~/utils/tauri"; +import { + MenuItem, + MenuItemList, + PopperContent, + topLeftAnimateClasses, +} from "../../editor/ui"; +import { SwitchButton } from "~/components/SwitchButton"; + +export default function RecordingSettings() { + const [store] = createResource(() => recordingSettingsStore.get()); + + return ( + + {(store) => } + + ); +} + +type SettingOption = { + label: string; + value: T; +}; + +type RecordingSetting = { + label: string; +} & ( + | { + key: "capture_resolution"; + options: SettingOption[]; + } + | { + key: "output_resolution"; + options: SettingOption[]; + } + | { + key: "recording_fps"; + options: SettingOption[]; + } +); + +const resolutionOptions: SettingOption[] = [ + { + label: "720p (1280x720)", + value: "_720p", + }, + { + label: "1080p (1920x1080)", + value: "_1080p", + }, + { + label: "4K (3840x2160)", + value: "_4K", + }, +]; + +const recordingSettings: RecordingSetting[] = [ + { + key: "capture_resolution", + label: "Screen capture resolution", + options: [ + { + label: "Same as display", + value: null, + }, + ...resolutionOptions, + ], + }, + { + key: "output_resolution", + label: "Output (scaled) resolution", + options: [ + { + label: "Same as captured", + value: null, + }, + ...resolutionOptions, + ], + }, + { + key: "recording_fps", + label: "Target FPS for screen capturing", + options: [ + { + label: "30 fps", + value: "_30", + }, + { + label: "60 fps", + value: "_60", + }, + ], + }, +]; + +function Inner(props: { initialStore: RecordingSettingsStore | null }) { + const currentRecording = createCurrentRecordingQuery(); + const [settings, setSettings] = createStore( + props.initialStore ?? { + use_hardware_acceleration: false, + capture_resolution: "_1080p", + output_resolution: null, + recording_fps: "_30", + } + ); + + const handleChange = async ( + key: K, + value: RecordingSettingsStore[K] + ) => { + console.log(`Handling settings change for ${key}: ${value}`); + + setSettings(key, value); + const result = await commands.setRecordingSettings({ + ...settings, + [key]: value, + }); + if (result.status === "error") console.error(result.error); + }; + + const settingOption = (setting: RecordingSetting) => { + return ( + setting.options.find( + (option) => option.value === settings[setting.key] + ) ?? null + ); + }; + + const recordingInProgress = !!currentRecording.data; + + return ( +
    +
    +
    +
    +
    +

    Use hardware acceleration

    + +
    + + {(setting) => ( +
    +

    {setting.label}

    + + options={setting.options} + optionValue="value" + optionTextValue="value" + value={settingOption(setting)} + disabled={recordingInProgress} + onChange={(option) => { + handleChange(setting.key, option?.value ?? null); + }} + allowDuplicateSelectionEvents={false} + itemComponent={(props) => ( + + as={KSelect.Item} + item={props.item} + > + + {props.item.rawValue.label} + + + )} + placeholder={setting.options[0].label} + > + + class=""> + {(state) => {state.selectedOption().label}} + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + + class="max-h-36 overflow-y-auto" + as={KSelect.Listbox} + /> + + + +
    + )} +
    +
    +
    +
    +
    + ); +} diff --git a/apps/desktop/src/store.ts b/apps/desktop/src/store.ts index ce3c2d40d..ddc9776fe 100644 --- a/apps/desktop/src/store.ts +++ b/apps/desktop/src/store.ts @@ -5,6 +5,7 @@ import type { ProjectConfiguration, HotkeysStore, GeneralSettingsStore, + RecordingSettingsStore, } from "~/utils/tauri"; let _store: Promise | undefined; @@ -64,3 +65,12 @@ export const generalSettingsStore = { await s.save(); }, }; + +export const recordingSettingsStore = { + get: () => store().then((s) => s.get("recording_settings")), + set: async (value: RecordingSettingsStore) => { + const s = await store(); + await s.set("recording_settings", value); + await s.save(); + } +}; diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index a16f2614f..d63ec5cd1 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -279,6 +279,14 @@ async setHotkey(action: HotkeyAction, hotkey: Hotkey | null) : Promise> { + try { + return { status: "ok", data: await TAURI_INVOKE("set_recording_settings", { settings }) }; +} catch (e) { + if(e instanceof Error) throw e; + else return { status: "error", error: e as any }; +} +}, async setGeneralSettings(settings: GeneralSettingsStore) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("set_general_settings", { settings }) }; @@ -411,6 +419,7 @@ export type RecordingMetaChanged = { id: string } export type RecordingOptions = { captureTarget: ScreenCaptureTarget; cameraLabel: string | null; audioInputName: string | null } export type RecordingOptionsChanged = null export type RecordingSegment = { start: number; end: number } +export type RecordingSettingsStore = { use_hardware_acceleration: boolean; capture_resolution?: TargetResolution | null; output_resolution?: TargetResolution | null; recording_fps: TargetFPS } export type RecordingStarted = null export type RecordingStopped = { path: string } export type RenderFrameEvent = { frame_number: number } @@ -425,10 +434,12 @@ export type ScreenCaptureTarget = ({ variant: "window" } & CaptureWindow) | ({ v export type SerializedEditorInstance = { framesSocketUrl: string; recordingDuration: number; savedProjectConfig: ProjectConfiguration; recordings: ProjectRecordings; path: string; prettyName: string } export type SharingMeta = { id: string; link: string } export type ShowCapturesPanel = null +export type TargetFPS = "_30" | "_60" +export type TargetResolution = "_720p" | "_1080p" | "_4K" export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments?: ZoomSegment[] } export type TimelineSegment = { timescale: number; start: number; end: number } export type UploadResult = { Success: string } | "NotAuthenticated" | "PlanCheckFailed" | "UpgradeRequired" -export type Video = { duration: number; width: number; height: number } +export type Video = { duration: number; width: number; height: number; fps: number } export type VideoType = "screen" | "output" export type XY = { x: T; y: T } export type ZoomSegment = { start: number; end: number; amount: number } diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index f0b8a414c..d669f252f 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -115,6 +115,7 @@ pub struct Renderer { rx: mpsc::Receiver, frame_tx: mpsc::UnboundedSender, render_constants: Arc, + fps: u32, } pub struct RendererHandle { @@ -125,6 +126,7 @@ impl Renderer { pub fn spawn( render_constants: Arc, frame_tx: mpsc::UnboundedSender, + fps: u32, ) -> RendererHandle { let (tx, rx) = mpsc::channel(4); @@ -132,6 +134,7 @@ impl Renderer { rx, frame_tx, render_constants, + fps, }; tokio::spawn(this.run()); @@ -173,6 +176,7 @@ impl Renderer { cap_rendering::Background::from(background), &uniforms, time, // Pass the actual time value + self.fps, ) .await .unwrap(); diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 16bb6987b..b16fe31ea 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -10,8 +10,6 @@ use std::sync::Mutex as StdMutex; use std::{path::PathBuf, sync::Arc}; use tokio::sync::{mpsc, watch, Mutex}; -const FPS: u32 = 30; - pub struct EditorInstance { pub project_path: PathBuf, pub id: String, @@ -95,7 +93,11 @@ impl EditorInstance { .unwrap(), ); - let renderer = Arc::new(editor::Renderer::spawn(render_constants.clone(), frame_tx)); + let renderer = Arc::new(editor::Renderer::spawn( + render_constants.clone(), + frame_tx, + recordings.display.fps, + )); let (preview_tx, preview_rx) = watch::channel(None); @@ -236,6 +238,8 @@ impl EditorInstance { self: Arc, mut preview_rx: watch::Receiver>, ) -> tokio::task::JoinHandle<()> { + let fps = self.recordings.display.fps; + tokio::spawn(async move { loop { preview_rx.changed().await.unwrap(); @@ -248,14 +252,14 @@ impl EditorInstance { let Some(time) = project .timeline .as_ref() - .map(|timeline| timeline.get_recording_time(frame_number as f64 / FPS as f64)) - .unwrap_or(Some(frame_number as f64 / FPS as f64)) + .map(|timeline| timeline.get_recording_time(frame_number as f64 / fps as f64)) + .unwrap_or(Some(frame_number as f64 / fps as f64)) else { continue; }; let Some((screen_frame, camera_frame)) = - self.decoders.get_frames((time * FPS as f64) as u32).await + self.decoders.get_frames((time * fps as f64) as u32).await else { continue; }; @@ -265,7 +269,7 @@ impl EditorInstance { screen_frame, camera_frame, project.background.source.clone(), - ProjectUniforms::new(&self.render_constants, &project, time as f32), + ProjectUniforms::new(&self.render_constants, &project, time as f32, fps), time as f32, // Add the time parameter ) .await; diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 0298acd26..f47cd12d4 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -22,8 +22,6 @@ pub struct Playback { pub recordings: ProjectRecordings, } -const FPS: u32 = 30; - #[derive(Clone, Copy)] pub enum PlaybackEvent { Start, @@ -53,6 +51,7 @@ impl Playback { tokio::spawn(async move { let start = Instant::now(); + let fps = self.recordings.display.fps; let mut frame_number = self.start_frame_number + 1; let duration = self @@ -68,35 +67,36 @@ impl Playback { audio: audio_data.clone(), stop_rx: stop_rx.clone(), start_frame_number: self.start_frame_number, - duration, + // duration, + fps, project: self.project.clone(), } .spawn(); }; loop { - if frame_number as f64 > FPS as f64 * duration { + if frame_number as f64 > fps as f64 * duration { break; }; let project = self.project.borrow().clone(); let time = if let Some(timeline) = project.timeline() { - match timeline.get_recording_time(frame_number as f64 / FPS as f64) { + match timeline.get_recording_time(frame_number as f64 / fps as f64) { Some(time) => time, None => break, } } else { - frame_number as f64 / FPS as f64 + frame_number as f64 / fps as f64 }; tokio::select! { _ = stop_rx.changed() => { break; }, - Some((screen_frame, camera_frame)) = self.decoders.get_frames((time * FPS as f64) as u32) => { + Some((screen_frame, camera_frame)) = self.decoders.get_frames((time * fps as f64) as u32) => { // println!("decoded frame in {:?}", debug.elapsed()); - let uniforms = ProjectUniforms::new(&self.render_constants, &project, time as f32); + let uniforms = ProjectUniforms::new(&self.render_constants, &project, time as f32, fps); self .renderer @@ -109,7 +109,7 @@ impl Playback { ) .await; - tokio::time::sleep_until(start + (frame_number - self.start_frame_number) * Duration::from_secs_f32(1.0 / FPS as f32)).await; + tokio::time::sleep_until(start + (frame_number - self.start_frame_number) * Duration::from_secs_f64(1.0 / fps as f64)).await; event_tx.send(PlaybackEvent::Frame(frame_number)).ok(); @@ -145,7 +145,8 @@ struct AudioPlayback { audio: AudioData, stop_rx: watch::Receiver, start_frame_number: u32, - duration: f64, + // duration: f64, + fps: u32, project: watch::Receiver, } @@ -194,17 +195,15 @@ impl AudioPlayback { stop_rx, start_frame_number, project, + fps, .. } = self; let mut output_info = AudioInfo::from_stream_config(&supported_config); output_info.sample_format = output_info.sample_format.packed(); - // TODO: Get fps and duration from video (once we start supporting other frame rates) - // Also, it's a bit weird that self.duration can ever be infinity to begin with, since - // pre-recorded videos are obviously a fixed size let mut audio_renderer = AudioPlaybackBuffer::new(audio, output_info); - let playhead = f64::from(start_frame_number) / f64::from(FPS); + let playhead = f64::from(start_frame_number) / fps as f64; audio_renderer.set_playhead(playhead, project.borrow().timeline()); // Prerender enough for smooth playback diff --git a/crates/editor/src/project_recordings.rs b/crates/editor/src/project_recordings.rs index 04906f6c9..2fcb26894 100644 --- a/crates/editor/src/project_recordings.rs +++ b/crates/editor/src/project_recordings.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use cap_project::RecordingMeta; +use cap_project::{RecordingMeta, TargetFPS}; use serde::Serialize; use specta::Type; @@ -9,6 +9,7 @@ pub struct Video { pub duration: f64, pub width: u32, pub height: u32, + pub fps: u32, } impl Video { @@ -26,6 +27,7 @@ impl Video { width: video_decoder.width(), height: video_decoder.height(), duration: input.duration() as f64 / 1_000_000.0, + fps: TargetFPS::round(stream.rate().into()), } } } diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 6e5b2d22e..046b05885 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -39,6 +39,8 @@ objc = "0.2.7" objc-foundation = "0.1.1" objc2-foundation = { version = "0.2.2", features = ["NSValue"] } nokhwa-bindings-macos.workspace = true +cidre = { git = "https://github.com/yury/cidre", rev = "1e008bec49a0f97aeaaea6130a0ba20fe00aa03b" } +screencapturekit = "0.2.8" [target.'cfg(target_os = "windows")'.dependencies] windows = { version = "0.52.0", features = [ diff --git a/crates/media/src/data.rs b/crates/media/src/data.rs index 131ce1190..26d24c5c5 100644 --- a/crates/media/src/data.rs +++ b/crates/media/src/data.rs @@ -210,24 +210,42 @@ impl VideoInfo { } } - pub fn scaled(&self, width: u32, fps: u32) -> Self { - let (width, height) = match self.width <= width { - true => (self.width, self.height), + pub fn with_resolution(&self, width: u32) -> Self { + match self.width <= width { + true => self.clone(), false => { let new_width = width & !1; let new_height = (((new_width as f32) * (self.height as f32) / (self.width as f32)) .round() as u32) & !1; - (new_width, new_height) + + Self { + width: new_width, + height: new_height, + ..self.clone() + } } - }; + } + } + pub fn with_fps(&self, fps: u32) -> Self { Self { - pixel_format: Pixel::NV12, - width, - height, - time_base: self.time_base, frame_rate: FFRational(fps.try_into().unwrap(), 1), + ..self.clone() + } + } + + pub fn with_software_format(&self) -> Self { + Self { + pixel_format: Pixel::YUV420P, + ..self.clone() + } + } + + pub fn with_hardware_format(&self) -> Self { + Self { + pixel_format: Pixel::NV12, + ..self.clone() } } diff --git a/crates/media/src/encoders/h264.rs b/crates/media/src/encoders/h264.rs index 819c8efba..fc4c355cf 100644 --- a/crates/media/src/encoders/h264.rs +++ b/crates/media/src/encoders/h264.rs @@ -6,6 +6,7 @@ use crate::{ use ffmpeg::{ codec::{codec::Codec, context, encoder}, format::{self}, + software, threading::Config, Dictionary, }; @@ -17,11 +18,13 @@ pub struct H264Encoder { encoder: encoder::Video, output_ctx: format::context::Output, last_pts: Option, + config: VideoInfo, } impl H264Encoder { pub fn init(tag: &'static str, config: VideoInfo, output: Output) -> Result { let Output::File(destination) = output; + let mut output_ctx = format::output(&destination)?; let (codec, options) = get_codec_and_options(&config)?; @@ -37,7 +40,14 @@ impl H264Encoder { encoder.set_format(config.pixel_format); encoder.set_time_base(config.frame_rate.invert()); encoder.set_frame_rate(Some(config.frame_rate)); - encoder.set_bit_rate(5_000_000); + + if codec.name() == "h264_videotoolbox" { + encoder.set_bit_rate(1_200_000); + encoder.set_max_bit_rate(120_000); + } else { + encoder.set_bit_rate(8_000_000); + encoder.set_max_bit_rate(8_000_000); + } let video_encoder = encoder.open_with(options)?; @@ -52,6 +62,7 @@ impl H264Encoder { encoder: video_encoder, output_ctx, last_pts: None, + config, }) } @@ -109,7 +120,9 @@ impl PipelineSinkTask for H264Encoder { fn get_codec_and_options(config: &VideoInfo) -> Result<(Codec, Dictionary), MediaError> { let encoder_name = { if cfg!(target_os = "macos") { - "h264_videotoolbox" + "libx264" + // looks terrible rn :( + // "h264_videotoolbox" } else { "libx264" } @@ -117,17 +130,22 @@ fn get_codec_and_options(config: &VideoInfo) -> Result<(Codec, Dictionary), Medi if let Some(codec) = encoder::find_by_name(encoder_name) { let mut options = Dictionary::new(); - let keyframe_interval_secs = 2; - let keyframe_interval = keyframe_interval_secs * config.frame_rate.numerator(); - let keyframe_interval_str = keyframe_interval.to_string(); - - options.set("preset", "ultrafast"); - options.set("tune", "zerolatency"); - options.set("vsync", "1"); - options.set("g", &keyframe_interval_str); - options.set("keyint_min", &keyframe_interval_str); - // TODO: Is it worth limiting quality? Maybe make this configurable - // options.set("crf", "23"); + if encoder_name == "h264_videotoolbox" { + // options.set("constant_bit_rate", "true"); + options.set("realtime", "true"); + } else { + let keyframe_interval_secs = 2; + let keyframe_interval = keyframe_interval_secs * config.frame_rate.numerator(); + let keyframe_interval_str = keyframe_interval.to_string(); + + options.set("preset", "ultrafast"); + options.set("tune", "zerolatency"); + options.set("vsync", "1"); + options.set("g", &keyframe_interval_str); + options.set("keyint_min", &keyframe_interval_str); + // // TODO: Is it worth limiting quality? Maybe make this configurable + // options.set("crf", "14"); + } return Ok((codec, options)); } diff --git a/crates/media/src/encoders/h264_avassetwriter.rs b/crates/media/src/encoders/h264_avassetwriter.rs new file mode 100644 index 000000000..49143d351 --- /dev/null +++ b/crates/media/src/encoders/h264_avassetwriter.rs @@ -0,0 +1,136 @@ +use crate::{data::VideoInfo, pipeline::task::PipelineSinkTask, MediaError}; + +use super::Output; +use arc::Retained; +use cidre::{objc::Obj, *}; + +pub struct H264AVAssetWriterEncoder { + tag: &'static str, + last_pts: Option, + config: Option, + asset_writer: Retained, + video_input: Retained, + first_timestamp: Option, + last_timestamp: Option, +} + +impl H264AVAssetWriterEncoder { + pub fn init( + tag: &'static str, + maybe_config: Option, + output: Output, + ) -> Result { + let Output::File(destination) = output; + + let mut asset_writer = av::AssetWriter::with_url_and_file_type( + cf::Url::with_path(&destination.as_path(), false) + .unwrap() + .as_ns(), + av::FileType::mp4(), + ) + .unwrap(); + + let assistant = + av::OutputSettingsAssistant::with_preset(av::OutputSettingsPreset::h264_3840x2160()) + .unwrap(); + + let mut output_settings = assistant.video_settings().unwrap().copy_mut(); + + if let Some(config) = maybe_config.as_ref() { + output_settings.insert( + av::video_settings_keys::width(), + ns::Number::with_u32(config.width).as_id_ref(), + ); + + output_settings.insert( + av::video_settings_keys::height(), + ns::Number::with_u32(config.height).as_id_ref(), + ); + } + + output_settings.insert( + av::video_settings_keys::compression_props(), + ns::Dictionary::with_keys_values( + &[unsafe { AVVideoAverageBitRateKey }], + &[ns::Number::with_u32(10_000_000).as_id_ref()], + ) + .as_id_ref(), + ); + + let mut video_input = av::AssetWriterInput::with_media_type_and_output_settings( + av::MediaType::video(), + Some(output_settings.as_ref()), + ) + .unwrap(); + video_input.set_expects_media_data_in_real_time(true); + + asset_writer.add_input(&video_input).unwrap(); + + asset_writer.start_writing(); + + Ok(Self { + tag, + last_pts: None, + config: maybe_config, + asset_writer, + video_input, + first_timestamp: None, + last_timestamp: None, + }) + } + + fn queue_frame(&mut self, frame: screencapturekit::cm_sample_buffer::CMSampleBuffer) { + let sample_buf = unsafe { + let ptr = &*frame.sys_ref as *const _ as *const cm::SampleBuf; + &*ptr + }; + + let time = sample_buf.pts(); + + if self.first_timestamp.is_none() { + self.asset_writer.start_session_at_src_time(time); + self.first_timestamp = Some(time); + } + + self.last_timestamp = Some(time); + + self.video_input.append_sample_buf(sample_buf).unwrap(); + } + + fn process_frame(&mut self) {} + + fn finish(&mut self) { + self.asset_writer + .end_session_at_src_time(self.last_timestamp.take().unwrap_or(cm::Time::zero())); + self.video_input.mark_as_finished(); + self.asset_writer.finish_writing(); + } +} + +impl PipelineSinkTask for H264AVAssetWriterEncoder { + type Input = screencapturekit::cm_sample_buffer::CMSampleBuffer; + + fn run( + &mut self, + ready_signal: crate::pipeline::task::PipelineReadySignal, + input: flume::Receiver, + ) { + println!("Starting {} video encoding thread", self.tag); + ready_signal.send(Ok(())).unwrap(); + + while let Ok(frame) = input.recv() { + self.queue_frame(frame); + self.process_frame(); + } + + println!("Received last {} frame. Finishing up encoding.", self.tag); + self.finish(); + + println!("Shutting down {} video encoding thread", self.tag); + } +} + +#[link(name = "AVFoundation", kind = "framework")] +extern "C" { + static AVVideoAverageBitRateKey: &'static cidre::ns::String; +} diff --git a/crates/media/src/encoders/mod.rs b/crates/media/src/encoders/mod.rs index bc184da53..27d9ba0cd 100644 --- a/crates/media/src/encoders/mod.rs +++ b/crates/media/src/encoders/mod.rs @@ -1,9 +1,13 @@ use std::path::PathBuf; mod h264; +#[cfg(target_os = "macos")] +mod h264_avassetwriter; mod mp3; pub use h264::*; +#[cfg(target_os = "macos")] +pub use h264_avassetwriter::*; pub use mp3::*; pub enum Output { diff --git a/crates/media/src/feeds/camera.rs b/crates/media/src/feeds/camera.rs index 10122efe6..757cde9f3 100644 --- a/crates/media/src/feeds/camera.rs +++ b/crates/media/src/feeds/camera.rs @@ -177,7 +177,7 @@ fn find_camera(selected_camera: &String) -> Result { fn create_camera(info: &CameraInfo) -> Result { dbg!(info); - // TODO: Make selected format more flexible + // TODO: Make selected format more flexible (also record at higher FPS maybe? Leaving it at 30 is fine for now) // let format = RequestedFormat::new::(RequestedFormatType::AbsoluteHighestResolution); let format = RequestedFormat::with_formats( RequestedFormatType::ClosestIgnoringFormat { diff --git a/crates/media/src/sources/screen_capture.rs b/crates/media/src/sources/screen_capture.rs index 277f5eb20..e15711494 100644 --- a/crates/media/src/sources/screen_capture.rs +++ b/crates/media/src/sources/screen_capture.rs @@ -1,4 +1,7 @@ use cap_flags::FLAGS; +use cap_project::{TargetFPS, TargetResolution}; +use cidre::cm; +use core_foundation::base::{kCFAllocatorDefault, CFAllocatorRef}; use flume::Sender; use scap::{ capturer::{get_output_frame_size, Area, Capturer, Options, Point, Resolution, Size}, @@ -7,15 +10,17 @@ use scap::{ }; use serde::{Deserialize, Serialize}; use specta::Type; -use std::collections::HashMap; +use std::{ + collections::HashMap, + ffi::c_void, + path::PathBuf, + ptr::{null, null_mut}, +}; use crate::{ data::{FFVideo, RawVideoFormat, VideoInfo}, - platform::{Bounds, Window}, -}; -use crate::{ pipeline::{clock::*, control::Control, task::PipelineSourceTask}, - platform, + platform::{self, Bounds, Window}, }; static EXCLUDED_WINDOWS: [&str; 4] = [ @@ -60,34 +65,35 @@ impl PartialEq for ScreenCaptureTarget { } } -pub struct ScreenCaptureSource { +pub struct ScreenCaptureSource { options: Options, video_info: VideoInfo, target: ScreenCaptureTarget, pub bounds: Bounds, + phantom: std::marker::PhantomData, } -impl ScreenCaptureSource { - pub const DEFAULT_FPS: u32 = 30; - +impl ScreenCaptureSource { pub fn init( capture_target: &ScreenCaptureTarget, - fps: Option, - resolution: Option, + fps: TargetFPS, + resolution: Option, ) -> Self { - let fps = fps.unwrap_or(Self::DEFAULT_FPS); - let output_resolution = resolution.unwrap_or(Resolution::Captured); + let fps = fps.to_raw(); + let output_resolution = resolution + .map(|target_resolution| match target_resolution { + TargetResolution::_720p => Resolution::_720p, + TargetResolution::_1080p => Resolution::_1080p, + TargetResolution::_4K => Resolution::_2160p, + }) + .unwrap_or(Resolution::Captured); let targets = dbg!(scap::get_all_targets()); let excluded_targets: Vec = targets .iter() - .filter(|target| match target { - Target::Window(scap_window) - if EXCLUDED_WINDOWS.contains(&scap_window.title.as_str()) => - { - true - } - _ => false, + .filter(|target| { + matches!(target, Target::Window(scap_window) + if EXCLUDED_WINDOWS.contains(&scap_window.title.as_str())) }) .cloned() .collect(); @@ -141,6 +147,7 @@ impl ScreenCaptureSource { target: capture_target.clone(), bounds, video_info: VideoInfo::from_raw(RawVideoFormat::Nv12, frame_width, frame_height, fps), + phantom: Default::default(), } } @@ -208,7 +215,9 @@ impl ScreenCaptureSource { } } -impl PipelineSourceTask for ScreenCaptureSource { +pub struct AVFrameCapture; + +impl PipelineSourceTask for ScreenCaptureSource { type Clock = RealTimeClock; type Output = FFVideo; @@ -317,3 +326,81 @@ impl PipelineSourceTask for ScreenCaptureSource { println!("Shutting down screen capture source thread."); } } + +#[cfg(target_os = "macos")] +pub struct CMSampleBufferCapture; + +#[cfg(target_os = "macos")] +impl PipelineSourceTask for ScreenCaptureSource { + type Clock = RealTimeClock; + type Output = screencapturekit::cm_sample_buffer::CMSampleBuffer; + + fn run( + &mut self, + _: Self::Clock, + ready_signal: crate::pipeline::task::PipelineReadySignal, + mut control_signal: crate::pipeline::control::PipelineControlSignal, + output: Sender, + ) { + use cidre::*; + + println!("Preparing screen capture source thread..."); + + let maybe_capture_window_id = match &self.target { + ScreenCaptureTarget::Window(window) => Some(window.id), + _ => None, + }; + let mut capturer = Capturer::new(dbg!(self.options.clone())); + let mut capturing = false; + ready_signal.send(Ok(())).unwrap(); + + loop { + match control_signal.last() { + Some(Control::Play) => { + if !capturing { + if let Some(window_id) = maybe_capture_window_id { + crate::platform::bring_window_to_focus(window_id); + } + capturer.start_capture(); + capturing = true; + + println!("Screen recording started."); + } + + match capturer.raw().get_next_pixel_buffer() { + Ok(pixel_buffer) => { + if pixel_buffer.height() == 0 || pixel_buffer.width() == 0 { + continue; + } + + if let Err(_) = output.send(pixel_buffer.into()) { + eprintln!("Pipeline is unreachable. Shutting down recording."); + break; + } + } + Err(error) => { + eprintln!("Capture error: {error}"); + break; + } + } + } + Some(Control::Pause) => { + println!("Received pause signal"); + if capturing { + capturer.stop_capture(); + capturing = false; + } + } + Some(Control::Shutdown) | None => { + println!("Received shutdown signal"); + if capturing { + capturer.stop_capture(); + } + break; + } + } + } + + println!("Shutting down screen capture source thread."); + } +} diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index c3df56a83..5ab42726c 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -51,6 +51,56 @@ impl Default for BackgroundSource { } } +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] +pub enum TargetResolution { + _720p, + #[default] + _1080p, + _4K, +} + +impl TargetResolution { + pub fn to_width(&self) -> u32 { + match self { + TargetResolution::_720p => 1280, + TargetResolution::_1080p => 1920, + TargetResolution::_4K => 3840, + } + } +} + +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] +pub enum TargetFPS { + #[default] + _30, + _60, +} + +impl TargetFPS { + pub fn to_raw(&self) -> u32 { + match self { + TargetFPS::_30 => 30, + TargetFPS::_60 => 60, + } + } + + pub fn round(fps: f64) -> u32 { + // Common monitor refresh rates + match fps.round() { + 29.0..31.0 => 30, + 59.0..61.0 => 60, + 74.0..76.0 => 75, + 89.0..91.0 => 90, + 119.0..121.0 => 120, + 143.0..145.0 => 144, + 164.0..166.0 => 165, + 239.0..241.0 => 240, + 359.0..361.0 => 360, + _ => unimplemented!("Unknown refresh rate"), + } + } +} + #[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] #[serde(rename_all = "camelCase")] pub struct XY { diff --git a/crates/rendering/src/decoder.rs b/crates/rendering/src/decoder.rs index 996e1065f..3fcd3ecbc 100644 --- a/crates/rendering/src/decoder.rs +++ b/crates/rendering/src/decoder.rs @@ -4,10 +4,11 @@ use std::{ sync::{mpsc, Arc}, }; +use cap_project::TargetFPS; use ffmpeg::{ codec, - format::{self, context::input::PacketIter}, - frame, rescale, Codec, Packet, Rational, Rescale, Stream, + format::{self}, + frame, rescale, Codec, Rational, Rescale, }; use ffmpeg_hw_device::{CodecContextExt, HwDevice}; use ffmpeg_sys_next::{avcodec_find_decoder, AVHWDeviceType}; @@ -18,13 +19,16 @@ enum VideoDecoderMessage { GetFrame(u32, tokio::sync::oneshot::Sender>>>), } -fn ts_to_frame(ts: i64, time_base: Rational, frame_rate: Rational) -> u32 { - // dbg!((ts, time_base, frame_rate)); - ((ts * time_base.numerator() as i64 * frame_rate.numerator() as i64) - / (time_base.denominator() as i64 * frame_rate.denominator() as i64)) as u32 +fn pts_to_frame(fps: f64, pts: i64, time_base: Rational) -> u32 { + (fps * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64))).round() + as u32 } const FRAME_CACHE_SIZE: usize = 50; +// TODO: Allow dynamic FPS values by either passing it into `spawn` +// or changing `get_frame` to take the requested time instead of frame number, +// so that the lookup can be done by PTS instead of frame number. +// const FPS: u32 = 30; pub struct AsyncVideoDecoder; @@ -50,6 +54,7 @@ impl AsyncVideoDecoder { let input_stream_index = input_stream.index(); let time_base = input_stream.time_base(); let frame_rate = input_stream.rate(); + let fps = TargetFPS::round(frame_rate.into()); // Create a decoder for the video stream let mut decoder = context.decoder().video().unwrap(); @@ -90,97 +95,114 @@ impl AsyncVideoDecoder { let mut temp_frame = ffmpeg::frame::Video::empty(); - let render_more_margin = (FRAME_CACHE_SIZE / 4) as u32; - let mut cache = BTreeMap::>>::new(); // active frame is a frame that triggered decode. // frames that are within render_more_margin of this frame won't trigger decode. let mut last_active_frame = None::; let mut last_decoded_frame = None::; - - struct PacketStuff<'a> { - packets: PacketIter<'a>, - skipped_packet: Option<(Stream<'a>, Packet)>, - } + let mut last_sent_frame = None::<(u32, DecodedFrame)>; let mut peekable_requests = PeekableReceiver { rx, peeked: None }; let mut packets = input.packets(); - // let mut packet_stuff = PacketStuff { - // packets: input.packets(), - // skipped_packet: None, - // }; while let Ok(r) = peekable_requests.recv() { match r { - VideoDecoderMessage::GetFrame(frame_number, sender) => { - // println!("retrieving frame {frame_number}"); - - let mut sender = if let Some(cached) = cache.get(&frame_number) { - // println!("sending frame {frame_number} from cache"); + VideoDecoderMessage::GetFrame(requested_frame, sender) => { + let mut sender = if let Some(cached) = cache.get(&requested_frame) { sender.send(Some(cached.clone())).ok(); + last_sent_frame = Some((requested_frame, cached.clone())); continue; } else { Some(sender) }; - let cache_min = frame_number.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); - let cache_max = frame_number + FRAME_CACHE_SIZE as u32 / 2; + let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); + let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; - if frame_number <= 0 - || last_decoded_frame - .map(|f| { - frame_number < f || + if requested_frame <= 0 + || last_sent_frame + .as_ref() + .map(|last| { + requested_frame < last.0 || // seek forward for big jumps. this threshold is arbitrary but should be derived from i-frames in future - frame_number - f > FRAME_CACHE_SIZE as u32 + requested_frame - last.0 > FRAME_CACHE_SIZE as u32 }) .unwrap_or(true) { let timestamp_us = - ((frame_number as f32 / frame_rate.numerator() as f32) + ((requested_frame as f32 / frame_rate.numerator() as f32) * 1_000_000.0) as i64; let position = timestamp_us.rescale((1, 1_000_000), rescale::TIME_BASE); - println!("seeking to {position} for frame {frame_number}"); + println!("seeking to {position} for frame {requested_frame}"); decoder.flush(); input.seek(position, ..position).unwrap(); cache.clear(); last_decoded_frame = None; + last_sent_frame = None; packets = input.packets(); } - last_active_frame = Some(frame_number); + // handle when requested_frame == last_decoded_frame or last_decoded_frame > requested_frame. + // the latter can occur when there are skips in frame numbers. + // in future we should alleviate this by using time + pts values instead of frame numbers. + if let Some((_, last_sent_frame)) = last_decoded_frame + .zip(last_sent_frame.as_ref()) + .filter(|(last_decoded_frame, last_sent_frame)| { + last_sent_frame.0 < requested_frame + && requested_frame < *last_decoded_frame + }) + { + if let Some(sender) = sender.take() { + sender.send(Some(last_sent_frame.1.clone())).ok(); + continue; + } + } + + last_active_frame = Some(requested_frame); loop { if peekable_requests.peek().is_some() { break; } let Some((stream, packet)) = packets.next() else { + sender.take().map(|s| s.send(None)); break; }; if stream.index() == input_stream_index { let start_offset = stream.start_time(); - let packet_frame = - ts_to_frame(packet.pts().unwrap(), time_base, frame_rate); - // println!("sending frame {packet_frame} packet"); decoder.send_packet(&packet).ok(); // decode failures are ok, we just fail to return a frame let mut exit = false; while decoder.receive_frame(&mut temp_frame).is_ok() { - let current_frame = ts_to_frame( + let current_frame = pts_to_frame( + fps as f64, temp_frame.pts().unwrap() - start_offset, time_base, - frame_rate, ); - // println!("processing frame {current_frame}"); + last_decoded_frame = Some(current_frame); + // we repeat the similar section as above to do the check per-frame instead of just per-request + if let Some((_, last_sent_frame)) = last_decoded_frame + .zip(last_sent_frame.as_ref()) + .filter(|(last_decoded_frame, last_sent_frame)| { + last_sent_frame.0 <= requested_frame + && requested_frame < *last_decoded_frame + }) + { + if let Some(sender) = sender.take() { + sender.send(Some(last_sent_frame.1.clone())).ok(); + } + } + let exceeds_cache_bounds = current_frame > cache_max; let too_small_for_cache_bounds = current_frame < cache_min; @@ -223,18 +245,22 @@ impl AsyncVideoDecoder { let frame = Arc::new(frame_buffer); - if current_frame == frame_number { + if current_frame == requested_frame { if let Some(sender) = sender.take() { + last_sent_frame = Some((current_frame, frame.clone())); sender.send(Some(frame.clone())).ok(); + + break; } } if !too_small_for_cache_bounds { if cache.len() >= FRAME_CACHE_SIZE { if let Some(last_active_frame) = &last_active_frame { - let frame = if frame_number > *last_active_frame { + let frame = if requested_frame > *last_active_frame + { *cache.keys().next().unwrap() - } else if frame_number < *last_active_frame { + } else if requested_frame < *last_active_frame { *cache.keys().next_back().unwrap() } else { let min = *cache.keys().min().unwrap(); @@ -265,8 +291,8 @@ impl AsyncVideoDecoder { } } - if sender.is_some() { - println!("failed to send frame {frame_number}"); + if let Some(s) = sender.take() { + let _ = s.send(None); } } } @@ -283,10 +309,10 @@ pub struct AsyncVideoDecoderHandle { } impl AsyncVideoDecoderHandle { - pub async fn get_frame(&self, frame_number: u32) -> Option>> { + pub async fn get_frame(&self, time: u32) -> Option>> { let (tx, rx) = tokio::sync::oneshot::channel(); self.sender - .send(VideoDecoderMessage::GetFrame(frame_number, tx)) + .send(VideoDecoderMessage::GetFrame(time, tx)) .unwrap(); rx.await.ok().flatten() } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 8b8561b0d..5b6339d36 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -119,6 +119,7 @@ pub async fn render_video_to_channel( decoders: RecordingDecoders, cursor: Arc, project_path: PathBuf, // Add project_path parameter + fps: u32, ) -> Result<(), String> { let constants = RenderVideoConstants::new(options, cursor, project_path).await?; @@ -136,23 +137,23 @@ pub async fn render_video_to_channel( let background = Background::from(project.background.source.clone()); loop { - if frame_number as f64 > 30_f64 * duration { + if frame_number as f64 > fps as f64 * duration { break; }; let time = if let Some(timeline) = project.timeline() { - match timeline.get_recording_time(frame_number as f64 / 30_f64) { + match timeline.get_recording_time(frame_number as f64 / fps as f64) { Some(time) => time, None => break, } } else { - frame_number as f64 / 30_f64 + frame_number as f64 / fps as f64 }; - let uniforms = ProjectUniforms::new(&constants, &project, time as f32); + let uniforms = ProjectUniforms::new(&constants, &project, time as f32, fps); let Some((screen_frame, camera_frame)) = - decoders.get_frames((time * 30.0) as u32).await + decoders.get_frames((time * fps as f64) as u32).await else { break; }; @@ -164,6 +165,7 @@ pub async fn render_video_to_channel( background, &uniforms, time as f32, + fps, ) .await { @@ -472,6 +474,7 @@ impl ProjectUniforms { constants: &RenderVideoConstants, project: &ProjectConfiguration, time: f32, + fps: u32, ) -> Self { let options = &constants.options; let output_size = Self::get_output_size(options, project); @@ -480,7 +483,7 @@ impl ProjectUniforms { let zoom_keyframes = ZoomKeyframes::new(project); let current_zoom = zoom_keyframes.get_amount(time as f64); - let prev_zoom = zoom_keyframes.get_amount((time - 1.0 / 30.0) as f64); + let prev_zoom = zoom_keyframes.get_amount(time as f64 - 1.0 / (fps as f64)); let velocity = if current_zoom != prev_zoom { let scale_change = (current_zoom - prev_zoom) as f32; @@ -633,7 +636,7 @@ impl ProjectUniforms { let zoom_delta = (current_zoom - prev_zoom).abs() as f32; // Calculate a smooth transition factor - let transition_speed = 30.0f32; // Frames per second + let transition_speed = fps as f32; // Frames per second let transition_factor = (zoom_delta * transition_speed).min(1.0); // Reduce multiplier from 3.0 to 2.0 for weaker blur @@ -757,6 +760,7 @@ pub async fn produce_frame( background: Background, uniforms: &ProjectUniforms, time: f32, + fps: u32, ) -> Result, String> { let mut encoder = constants.device.create_command_encoder( &(wgpu::CommandEncoderDescriptor { @@ -886,6 +890,7 @@ pub async fn produce_frame( constants, uniforms, time, + fps, &mut encoder, get_either(texture_views, !output_is_left), ); @@ -1043,6 +1048,7 @@ fn draw_cursor( constants: &RenderVideoConstants, uniforms: &ProjectUniforms, time: f32, + fps: u32, encoder: &mut CommandEncoder, view: &wgpu::TextureView, ) { @@ -1051,7 +1057,7 @@ fn draw_cursor( }; // Calculate previous position for velocity - let prev_position = interpolate_cursor_position(&constants.cursor, time - 1.0 / 30.0); + let prev_position = interpolate_cursor_position(&constants.cursor, time - 1.0 / (fps as f32)); // Calculate velocity in screen space let velocity = if let Some(prev_pos) = prev_position {