letsplay_runner_core: Working encoding!

we can actually encode frames from a Game woohoo!!!!!! lesgo!!!!!!!!

crates/letsplay_av_ffmpeg/src/encoder_thread/hardware/nvenc.rs

@@ -204,6 +204,8 @@ fn main(
     // allocated.
     let mut temp_buffer: CudaSlice<u32> = cuda_device.alloc_zeros::<u32>(48).expect("over");
 
+	tracing::info!("Encoder thread ready for service!");
+
     loop {
         // wait for a message
         {

crates/letsplay_av_ffmpeg/src/video_encoder.rs

@@ -42,8 +42,9 @@ fn create_context_and_set_common_parameters(
 	video_encoder_context.set_max_bit_rate(bitrate);
 
 	// qp TODO:
-	//video_encoder_context.set_qmax(30);
-	//video_encoder_context.set_qmin(35);
+	video_encoder_context.set_qmin(30);
+	video_encoder_context.set_qmax(28);
+
 
 	video_encoder_context.set_time_base(ffmpeg::Rational(1, max_framerate as i32).invert());
 	video_encoder_context.set_format(ffmpeg::format::Pixel::YUV420P);
@@ -67,12 +68,8 @@ pub enum VideoEncoder {
 		encoder: ffmpeg::encoder::video::Encoder,
 	},
 
-	// FIXME: Rename this to `HardwareWithHwFrame`
-	// once we have multiple hardware encoding paths,
-	// and have sufficiently expanded them to support
-	// non-NVENC stuff.
-	/// Hardware encoding, with frames already on the GPU.
-	NvencHWFrame {
+	/// Hardware encoding, with frames already on or uploaded to the GPU.
+	HardwareWithHardwareFrame {
 		encoder: ffmpeg::encoder::video::Encoder,
 		hw_context: HwFrameContext,
 	},
@@ -160,9 +157,6 @@ impl VideoEncoder {
 
 		video_encoder_context.set_format(ffmpeg::format::Pixel::CUDA);
 
-		video_encoder_context.set_qmin(35);
-		video_encoder_context.set_qmax(38);
-
 		unsafe {
 			// FIXME: this currently breaks the avbufferref system a bit
 			(*video_encoder_context.as_mut_ptr()).hw_frames_ctx =
@@ -192,7 +186,7 @@ impl VideoEncoder {
 			.open_as_with(encoder, dict)
 			.with_context(|| "While opening h264_nvenc video codec")?;
 
-		Ok(Self::NvencHWFrame {
+		Ok(Self::HardwareWithHardwareFrame {
 			encoder: encoder,
 			hw_context: hw_frame_context,
 		})
@@ -203,7 +197,7 @@ impl VideoEncoder {
 	pub fn is_hardware(&mut self) -> bool {
 		match self {
 			Self::Software { .. } => false,
-			Self::NvencHWFrame { .. } => true,
+			Self::HardwareWithHardwareFrame { .. } => true,
 		}
 	}
 
@@ -224,7 +218,7 @@ impl VideoEncoder {
 				));
 			}
 
-			Self::NvencHWFrame {
+			Self::HardwareWithHardwareFrame {
 				encoder,
 				hw_context,
 			} => {
@@ -254,7 +248,7 @@ impl VideoEncoder {
 				encoder.send_frame(frame).unwrap();
 			}
 
-			Self::NvencHWFrame {
+			Self::HardwareWithHardwareFrame {
 				encoder,
 				hw_context: _,
 			} => {
@@ -269,7 +263,7 @@ impl VideoEncoder {
 				encoder.send_eof().unwrap();
 			}
 
-			Self::NvencHWFrame {
+			Self::HardwareWithHardwareFrame {
 				encoder,
 				hw_context: _,
 			} => {
@@ -281,7 +275,7 @@ impl VideoEncoder {
 	fn receive_packet_impl(&mut self, packet: &mut ffmpeg::Packet) -> Result<(), ffmpeg::Error> {
 		return match self {
 			Self::Software { encoder } => encoder.receive_packet(packet),
-			Self::NvencHWFrame {
+			Self::HardwareWithHardwareFrame {
 				encoder,
 				hw_context: _,
 			} => encoder.receive_packet(packet),

crates/letsplay_runner_core/src/client/game.rs

@@ -1,11 +1,13 @@
+use letsplay_av_ffmpeg::encoder_thread::EncoderThreadControl;
+
 use super::GraphicsContexts;
 use std::time::Instant;
 
 /// A game that should be run by the runner core.
 /// A runner implementation implements this trait.
 pub trait Game {
 	/// Do any initalization tasks. Graphics contexts are provided
-	fn init(&mut self, graphics_contexts: &GraphicsContexts);
+	fn init(&mut self, graphics_contexts: &GraphicsContexts, encoder_control: &EncoderThreadControl);
 
 	fn reset(&mut self);
 

crates/letsplay_runner_core/src/client/game_thread.rs

@@ -1,7 +1,5 @@
 use std::{
-	sync::{Arc, Mutex},
-	thread::{self, JoinHandle},
-	time::{Duration, Instant},
+	io::Write, sync::{Arc, Mutex}, thread::{self, JoinHandle}, time::{Duration, Instant}
 };
 
 // This is used by async code, so we have to use
@@ -10,6 +8,8 @@ use tokio::sync::mpsc::{self, error::TryRecvError};
 
 use super::{Game, GraphicsContexts};
 
+use letsplay_av_ffmpeg::encoder_thread;
+
 enum GameThreadMessage {
 	/// Shut down the game thread.
 	Shutdown,
@@ -37,10 +37,46 @@ fn main(mut rx: mpsc::UnboundedReceiver<GameThreadMessage>, mut game: Box<dyn Ga
 
 	let contexts = GraphicsContexts::create(0);
 
-	game.init(&contexts);
-
 	// Spawn the video thread here
 
+	let video_encoder_control = {
+		#[cfg(feature = "av-nvidia")]
+		{
+			encoder_thread::hardware::nvenc::spawn(
+				&contexts.cuda_context.clone(),
+				&contexts.cuda_interop_context.clone(),
+				&contexts.egl_device_context.clone(),
+				false,
+			)
+		}
+	};
+
+	game.init(&contexts, &video_encoder_control);
+
+	// TEMP CODE: This accepts a single tcp connection and broadcasts packets to it
+	// this is temporary as all hell
+	// yes I know sync io but its running on another thread anyways so its fine.
+	let server = std::net::TcpListener::bind("0.0.0.0:6040").expect("rrr");
+	let mut clients = Vec::new();
+
+	while clients.len() != 2 {
+		let client = server.accept().expect("baned");
+		clients.push(client.0);
+	}
+
+	tracing::info!("all clients accepted - unblocking and completing intialization");
+
+
+	let clone = video_encoder_control.clone();
+	std::thread::spawn(move || {
+		loop {
+			let frame = clone.wait_for_packet();
+			for client in &mut clients {
+				let _ = client.write_all(frame.data().unwrap());
+			}
+		}
+	});
+
 	loop {
 		match rx.try_recv() {
 			Ok(message) => match message {
@@ -88,11 +124,10 @@ fn main(mut rx: mpsc::UnboundedReceiver<GameThreadMessage>, mut game: Box<dyn Ga
 			lk.release();
 		}
 
-		// FIXME: Submit rendered frame to video thread,
-		// unless a frame is duplicated. (this allows us to hold output/do
-		// dynamic fps for static/mostly static scenes, which will *heavily* decrease bandwidth consumption
-		// on both the server and player ends)
-		//
+		// Tell the encoder thread to encode the frame we just ran
+		video_encoder_control.send_command(encoder_thread::EncoderCommand::SendFrame);
+
+		// FIXME: Output audio
 		// Audio should always be submitted and output (Opus supports DTX which would give us similar wins to frame duplication,
 		// but I'm not sure if the latency trade off is that worth it for a few kpbs less bandwidth)
 

crates/letsplay_runner_retro/Cargo.toml

@@ -10,6 +10,8 @@ letsplay_core.path = "../letsplay_core"
 letsplay_gpu.path = "../letsplay_gpu"
 letsplay_runner_core.path = "../letsplay_runner_core"
 
+letsplay_av_ffmpeg.path = "../letsplay_av_ffmpeg"
+
 # The important part (libretro frontend goodness)
 letsplay_retro_frontend.path = "../letsplay_retro_frontend"
 

crates/letsplay_runner_retro/src/main.rs

@@ -6,32 +6,42 @@ use std::{
 
 use client::GraphicsContexts;
 use letsplay_core::sleep;
-use letsplay_gpu::{self as gpu, egl_helpers::DeviceContext};
+use letsplay_gpu::{self as gpu, egl_helpers::DeviceContext, GlFramebuffer};
 use letsplay_retro_frontend::{
 	frontend::{Frontend, FrontendInterface, HwGlInitData},
 	input_devices::AnyDevice,
 };
 use letsplay_runner_core::*;
 
+use letsplay_av_ffmpeg::encoder_thread::EncoderCommand;
+use letsplay_av_ffmpeg::encoder_thread::EncoderThreadControl;
+
 /// Libretro game. very much TODO
 pub struct RetroGame {
 	graphics_contexts: Option<GraphicsContexts>,
+	encoder_control: Option<EncoderThreadControl>,
 
-	devices: BTreeMap<i32, AnyDevice>,
+	input_devices: BTreeMap<i32, AnyDevice>,
 
 	frontend: Option<Box<Frontend>>,
 
 	// timing
 	frame_duration: Duration,
+
+	// gl styff
+	gl_framebuffer: GlFramebuffer,
 }
 
 impl RetroGame {
 	fn new() -> Box<Self> {
 		let mut s = Box::new(Self {
 			graphics_contexts: None,
-			devices: BTreeMap::new(),
+			encoder_control: None,
+			input_devices: BTreeMap::new(),
 			frontend: None,
 			frame_duration: Duration::new(0, 0),
+
+			gl_framebuffer: GlFramebuffer::new(),
 		});
 
 		// SAFETY: The only way to touch the pointer involves the frontend library calling retro_run,
@@ -52,7 +62,11 @@ impl RetroGame {
 }
 
 impl client::Game for RetroGame {
-	fn init(&mut self, graphics_contexts: &client::GraphicsContexts) {
+	fn init(
+		&mut self,
+		graphics_contexts: &client::GraphicsContexts,
+		encoder_control: &EncoderThreadControl,
+	) {
 		// HACK: Make the context current when we reach init() because
 		// libretro assumes you keep the context current when loading the game
 		// Once unsuspended (and we're actually context sharing) we will
@@ -64,6 +78,7 @@ impl client::Game for RetroGame {
 
 		// Scary but these are all Arc<> pointers anyways so its not a big deal
 		self.graphics_contexts = Some(graphics_contexts.clone());
+		self.encoder_control = Some(encoder_control.clone());
 	}
 
 	fn reset(&mut self) {
@@ -74,7 +89,7 @@ impl client::Game for RetroGame {
 		match key {
 			"libretro.core" => {
 				tracing::info!("Core is {value}");
-				// TODO: Failure should be logged!
+				// TODO: Failure should be logged, not panic worthy
 				self.get_frontend()
 					.load_core(value)
 					.expect("Failed to load core");
@@ -106,6 +121,40 @@ impl client::Game for RetroGame {
 impl FrontendInterface for RetroGame {
 	fn video_resize(&mut self, width: u32, height: u32) {
 		tracing::info!("Resized to {width}x{height}");
+		self.gl_framebuffer.resize(width, height);
+		let raw = self.gl_framebuffer.as_raw();
+
+		// Notify the frontend layer about the new FBO ID
+		self.get_frontend().set_gl_fbo(raw);
+
+		// register the FBO's texture to our cuda interop resource
+		#[cfg(feature = "av-nvidia")]
+		{
+			let mut locked = self
+				.graphics_contexts
+				.as_ref()
+				.unwrap()
+				.cuda_interop_context
+				.lock()
+				.expect("Failed to lock CUDA resource");
+
+			locked
+				.device()
+				.bind_to_thread()
+				.expect("Failed to bind CUDA device to thread");
+
+			locked
+				.register(self.gl_framebuffer.texture_id(), gl::TEXTURE_2D)
+				.expect("Failed to register OpenGL texture with CUDA Graphics resource");
+		}
+
+		// FIXME: Not this
+		self.encoder_control
+			.as_ref()
+			.unwrap()
+			.send_command(EncoderCommand::Init {
+				size: letsplay_core::Size { width, height },
+			});
 	}
 
 	fn video_update(&mut self, slice: &[u32], pitch: u32) {
@@ -125,10 +174,6 @@ impl FrontendInterface for RetroGame {
 				let str = std::ffi::CString::new(s).expect("gl::load_with fail");
 				std::mem::transmute(gpu::egl::GetProcAddress(str.as_ptr()))
 			});
-
-			// set OpenGL debug message callback
-			//gl::Enable(gl::DEBUG_OUTPUT);
-			//gl::DebugMessageCallback(Some(opengl_message_callback), std::ptr::null());
 		}
 
 		return Some(HwGlInitData {