Skip to content

Commit

Permalink
opt qos (rustdesk#10459)
Browse files Browse the repository at this point in the history 
* Adjust bitrate and fps based on TestDelay messages.
* Bitrate is adjusted every 3 seconds, fps is adjusted every second and when receiving test lag.
* Latency optimized at high resolutions. However, when the network is poor, the delay when just connecting or sliding static pages is still obvious.

Signed-off-by: 21pages <[email protected]>
  • Loading branch information
@21pages
21pages authored Jan 20, 2025
1 parent c44803f commit 5fa8c25
Show file tree
Hide file tree
Showing 10 changed files with 719 additions and 532 deletions.
27 changes: 7 additions & 20 deletions libs/scrap/examples/benchmark.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use hbb_common::{
};
use scrap::{
aom::{AomDecoder, AomEncoder, AomEncoderConfig},
codec::{EncoderApi, EncoderCfg, Quality as Q},
codec::{EncoderApi, EncoderCfg},
Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig,
VpxVideoCodecId::{self, *},
STRIDE_ALIGN,
Expand All @@ -27,25 +27,17 @@ Usage:
Options:
-h --help Show this screen.
--count=COUNT Capture frame count [default: 100].
--quality=QUALITY Video quality [default: Balanced].
Valid values: Best, Balanced, Low.
--quality=QUALITY Video quality [default: 1.0].
--i444 I444.
";

#[derive(Debug, serde::Deserialize, Clone, Copy)]
struct Args {
flag_count: usize,
flag_quality: Quality,
flag_quality: f32,
flag_i444: bool,
}

#[derive(Debug, serde::Deserialize, Clone, Copy)]
enum Quality {
Best,
Balanced,
Low,
}

fn main() {
init_from_env(Env::default().filter_or(DEFAULT_FILTER_ENV, "info"));
let args: Args = Docopt::new(USAGE)
Expand All @@ -70,11 +62,6 @@ fn main() {
"benchmark {}x{} quality:{:?}, i444:{:?}",
width, height, quality, args.flag_i444
);
let quality = match quality {
Quality::Best => Q::Best,
Quality::Balanced => Q::Balanced,
Quality::Low => Q::Low,
};
[VP8, VP9].map(|codec| {
test_vpx(
&mut c,
Expand All @@ -98,7 +85,7 @@ fn test_vpx(
codec_id: VpxVideoCodecId,
width: usize,
height: usize,
quality: Q,
quality: f32,
yuv_count: usize,
i444: bool,
) {
Expand Down Expand Up @@ -177,7 +164,7 @@ fn test_av1(
c: &mut Capturer,
width: usize,
height: usize,
quality: Q,
quality: f32,
yuv_count: usize,
i444: bool,
) {
Expand Down Expand Up @@ -247,7 +234,7 @@ mod hw {

use super::*;

pub fn test(c: &mut Capturer, width: usize, height: usize, quality: Q, yuv_count: usize) {
pub fn test(c: &mut Capturer, width: usize, height: usize, quality: f32, yuv_count: usize) {
let mut h264s = Vec::new();
let mut h265s = Vec::new();
if let Some(info) = HwRamEncoder::try_get(CodecFormat::H264) {
Expand All @@ -263,7 +250,7 @@ mod hw {
fn test_encoder(
width: usize,
height: usize,
quality: Q,
quality: f32,
info: CodecInfo,
c: &mut Capturer,
yuv_count: usize,
Expand Down
20 changes: 4 additions & 16 deletions libs/scrap/examples/record-screen.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use std::time::{Duration, Instant};
use std::{io, thread};

use docopt::Docopt;
use scrap::codec::{EncoderApi, EncoderCfg, Quality as Q};
use scrap::codec::{EncoderApi, EncoderCfg};
use webm::mux;
use webm::mux::Track;

Expand All @@ -31,8 +31,7 @@ Options:
-h --help Show this screen.
--time=<s> Recording duration in seconds.
--fps=<fps> Frames per second [default: 30].
--quality=<quality> Video quality [default: Balanced].
Valid values: Best, Balanced, Low.
--quality=<quality> Video quality [default: 1.0].
--ba=<kbps> Audio bitrate in kilobits per second [default: 96].
--codec CODEC Configure the codec used. [default: vp9]
Valid values: vp8, vp9.
Expand All @@ -44,14 +43,7 @@ struct Args {
flag_codec: Codec,
flag_time: Option<u64>,
flag_fps: u64,
flag_quality: Quality,
}

#[derive(Debug, serde::Deserialize)]
enum Quality {
Best,
Balanced,
Low,
flag_quality: f32,
}

#[derive(Debug, serde::Deserialize)]
Expand Down Expand Up @@ -105,11 +97,7 @@ fn main() -> io::Result<()> {
let mut vt = webm.add_video_track(width, height, None, mux_codec);

// Setup the encoder.
let quality = match args.flag_quality {
Quality::Best => Q::Best,
Quality::Balanced => Q::Balanced,
Quality::Low => Q::Low,
};
let quality = args.flag_quality;
let mut vpx = vpx_encode::VpxEncoder::new(
EncoderCfg::VPX(vpx_encode::VpxEncoderConfig {
width,
Expand Down
74 changes: 22 additions & 52 deletions libs/scrap/src/common/aom.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs"));

use crate::codec::{base_bitrate, codec_thread_num, Quality};
use crate::codec::{base_bitrate, codec_thread_num};
use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN};
use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result};
use crate::{EncodeInput, EncodeYuvFormat, Pixfmt};
Expand Down Expand Up @@ -45,7 +45,7 @@ impl Default for aom_image_t {
pub struct AomEncoderConfig {
pub width: u32,
pub height: u32,
pub quality: Quality,
pub quality: f32,
pub keyframe_interval: Option<usize>,
}

Expand All @@ -62,15 +62,9 @@ mod webrtc {
use super::*;

const kUsageProfile: u32 = AOM_USAGE_REALTIME;
const kMinQindex: u32 = 145; // Min qindex threshold for QP scaling.
const kMaxQindex: u32 = 205; // Max qindex threshold for QP scaling.
const kBitDepth: u32 = 8;
const kLagInFrames: u32 = 0; // No look ahead.
pub(super) const kTimeBaseDen: i64 = 1000;
const kMinimumFrameRate: f64 = 1.0;

pub const DEFAULT_Q_MAX: u32 = 56; // no more than 63
pub const DEFAULT_Q_MIN: u32 = 12; // no more than 63, litter than q_max

// Only positive speeds, range for real-time coding currently is: 6 - 8.
// Lower means slower/better quality, higher means fastest/lower quality.
Expand Down Expand Up @@ -116,21 +110,10 @@ mod webrtc {
} else {
c.kf_mode = aom_kf_mode::AOM_KF_DISABLED;
}
let (q_min, q_max, b) = AomEncoder::convert_quality(cfg.quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
} else {
c.rc_min_quantizer = DEFAULT_Q_MIN;
c.rc_max_quantizer = DEFAULT_Q_MAX;
}
let base_bitrate = base_bitrate(cfg.width as _, cfg.height as _);
let bitrate = base_bitrate * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
} else {
c.rc_target_bitrate = base_bitrate;
}
let (q_min, q_max) = AomEncoder::calc_q_values(cfg.quality);
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
c.rc_target_bitrate = AomEncoder::bitrate(cfg.width as _, cfg.height as _, cfg.quality);
c.rc_undershoot_pct = 50;
c.rc_overshoot_pct = 50;
c.rc_buf_initial_sz = 600;
Expand Down Expand Up @@ -273,17 +256,12 @@ impl EncoderApi for AomEncoder {
false
}

fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
fn set_quality(&mut self, ratio: f32) -> ResultType<()> {
let mut c = unsafe { *self.ctx.config.enc.to_owned() };
let (q_min, q_max, b) = Self::convert_quality(quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
}
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
}
let (q_min, q_max) = Self::calc_q_values(ratio);
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
c.rc_target_bitrate = Self::bitrate(self.width as _, self.height as _, ratio);
call_aom!(aom_codec_enc_config_set(&mut self.ctx, &c));
Ok(())
}
Expand All @@ -293,10 +271,6 @@ impl EncoderApi for AomEncoder {
c.rc_target_bitrate
}

fn support_abr(&self) -> bool {
true
}

fn support_changing_quality(&self) -> bool {
true
}
Expand Down Expand Up @@ -370,31 +344,27 @@ impl AomEncoder {
}
}

pub fn convert_quality(quality: Quality) -> (u32, u32, u32) {
// we can use lower bitrate for av1
match quality {
Quality::Best => (12, 25, 100),
Quality::Balanced => (12, 35, 100 * 2 / 3),
Quality::Low => (18, 45, 50),
Quality::Custom(b) => {
let (q_min, q_max) = Self::calc_q_values(b);
(q_min, q_max, b)
}
}
fn bitrate(width: u32, height: u32, ratio: f32) -> u32 {
let bitrate = base_bitrate(width, height) as f32;
(bitrate * ratio) as u32
}

#[inline]
fn calc_q_values(b: u32) -> (u32, u32) {
fn calc_q_values(ratio: f32) -> (u32, u32) {
let b = (ratio * 100.0) as u32;
let b = std::cmp::min(b, 200);
let q_min1: i32 = 24;
let q_min1 = 24;
let q_min2 = 5;
let q_max1 = 45;
let q_max2 = 25;

let t = b as f32 / 200.0;

let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
let mut q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
let mut q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;

q_min = q_min.clamp(q_min2, q_min1);
q_max = q_max.clamp(q_max2, q_max1);

(q_min, q_max)
}
Expand Down
68 changes: 53 additions & 15 deletions libs/scrap/src/common/codec.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,12 +62,10 @@ pub trait EncoderApi {
#[cfg(feature = "vram")]
fn input_texture(&self) -> bool;

fn set_quality(&mut self, quality: Quality) -> ResultType<()>;
fn set_quality(&mut self, ratio: f32) -> ResultType<()>;

fn bitrate(&self) -> u32;

fn support_abr(&self) -> bool;

fn support_changing_quality(&self) -> bool;

fn latency_free(&self) -> bool;
Expand Down Expand Up @@ -882,12 +880,16 @@ pub fn enable_directx_capture() -> bool {
)
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub const BR_BEST: f32 = 1.5;
pub const BR_BALANCED: f32 = 0.67;
pub const BR_SPEED: f32 = 0.5;

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Quality {
Best,
Balanced,
Low,
Custom(u32),
Custom(f32),
}

impl Default for Quality {
Expand All @@ -903,22 +905,59 @@ impl Quality {
_ => false,
}
}

pub fn ratio(&self) -> f32 {
match self {
Quality::Best => BR_BEST,
Quality::Balanced => BR_BALANCED,
Quality::Low => BR_SPEED,
Quality::Custom(v) => *v,
}
}
}

pub fn base_bitrate(width: u32, height: u32) -> u32 {
#[allow(unused_mut)]
let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9
if base_bitrate == 0 {
base_bitrate = 1920 * 1080 / 1000;
}
const RESOLUTION_PRESETS: &[(u32, u32, u32)] = &[
(640, 480, 400), // VGA, 307k pixels
(800, 600, 500), // SVGA, 480k pixels
(1024, 768, 800), // XGA, 786k pixels
(1280, 720, 1000), // 720p, 921k pixels
(1366, 768, 1100), // HD, 1049k pixels
(1440, 900, 1300), // WXGA+, 1296k pixels
(1600, 900, 1500), // HD+, 1440k pixels
(1920, 1080, 2073), // 1080p, 2073k pixels
(2048, 1080, 2200), // 2K DCI, 2211k pixels
(2560, 1440, 3000), // 2K QHD, 3686k pixels
(3440, 1440, 4000), // UWQHD, 4953k pixels
(3840, 2160, 5000), // 4K UHD, 8294k pixels
(7680, 4320, 12000), // 8K UHD, 33177k pixels
];
let pixels = width * height;

let (preset_pixels, preset_bitrate) = RESOLUTION_PRESETS
.iter()
.map(|(w, h, bitrate)| (w * h, bitrate))
.min_by_key(|(preset_pixels, _)| {
if *preset_pixels >= pixels {
preset_pixels - pixels
} else {
pixels - preset_pixels
}
})
.unwrap_or(((1920 * 1080) as u32, &2073)); // default 1080p

let bitrate = (*preset_bitrate as f32 * (pixels as f32 / preset_pixels as f32)).round() as u32;

#[cfg(target_os = "android")]
{
// fix when android screen shrinks
let fix = crate::Display::fix_quality() as u32;
log::debug!("Android screen, fix quality:{}", fix);
base_bitrate = base_bitrate * fix;
bitrate * fix
}
#[cfg(not(target_os = "android"))]
{
bitrate
}
base_bitrate
}

pub fn codec_thread_num(limit: usize) -> usize {
Expand Down Expand Up @@ -1001,8 +1040,7 @@ pub fn test_av1() {
static ONCE: Once = Once::new();
ONCE.call_once(|| {
let f = || {
let (width, height, quality, keyframe_interval, i444) =
(1920, 1080, Quality::Balanced, None, false);
let (width, height, quality, keyframe_interval, i444) = (1920, 1080, 1.0, None, false);
let frame_count = 10;
let block_size = 300;
let move_step = 50;
Expand Down
Loading

0 comments on commit 5fa8c25

Please sign in to comment.