From 316553f5fbeadee486efa0a54908ee3804833125 Mon Sep 17 00:00:00 2001 From: Ash Beitz <8304894+ashbeitz@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:58:57 -0700 Subject: [PATCH] Replace show image crate (#55) * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate * Replace show-image crate --- .accepted_words.txt | 3 + .github/workflows/rust-ci.yml | 8 +- Cargo.toml | 4 +- README.md | 8 +- samples/common/Cargo.toml | 2 + samples/common/src/image_rendering.rs | 105 +++++++++++++++++++++++++ samples/common/src/lib.rs | 1 + samples/streaming/Cargo.toml | 2 +- samples/streaming/consumer/src/main.rs | 43 +++++----- 9 files changed, 144 insertions(+), 32 deletions(-) create mode 100644 samples/common/src/image_rendering.rs diff --git a/.accepted_words.txt b/.accepted_words.txt index d34ad96a..2e4aab3d 100644 --- a/.accepted_words.txt +++ b/.accepted_words.txt @@ -36,6 +36,7 @@ JSON ld LD libfontconfig +libsdl microsoft minimalistic mosquitto @@ -50,6 +51,8 @@ repo Repo rustup sdk +sdl +SDL snapd sudo timothee diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 6e7ea40e..08e6c9b6 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -19,8 +19,8 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive - - name: Install protobuf-compiler - run: sudo apt-get install -y protobuf-compiler + - name: Install packages + run: sudo apt-get install -y protobuf-compiler libsdl2-dev - name: Install .NET 7.0 uses: actions/setup-dotnet@v3 with: @@ -53,8 +53,8 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive - - name: Install protobuf-compiler - run: sudo apt-get install -y protobuf-compiler + - name: Install packages + run: sudo apt-get install -y protobuf-compiler libsdl2-dev - name: Install .NET 7.0 uses: actions/setup-dotnet@v3 with: diff --git a/Cargo.toml b/Cargo.toml index 43136353..0b1755ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ members = [ "samples/mixed", "samples/property", "samples/seat_massager", - # "samples/streaming", + "samples/streaming", ] [workspace.dependencies] @@ -56,10 +56,10 @@ parking_lot = "0.12.1" prost = "0.12" prost-types = "0.12" regex = " 1.9.3" +sdl2 = "0.35.2" serde = "1.0.160" serde_derive = "1.0.163" serde_json = "^1.0" -show-image = "0.13.1" strum = "0.25" strum_macros = "0.25.1" tokio = "1.29.1" diff --git a/README.md b/README.md index 81656c9f..c25016ca 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ - [Install gcc](#install-gcc) - [Install Rust](#install-rust) - [Install Protobuf Compiler](#install-protobuf-compiler) - - [Install fontconfig-dev library](#install-fontconfig-dev-library) + - [Install SDL2 library](#install-sdl2-library) - [Install MQTT Broker](#install-mqtt-broker) - [Cloning the Repo](#cloning-the-repo) - [Developer Notes](#developer-notes) @@ -66,12 +66,12 @@ You will need to install the Protobuf Compiler. This can be done by executing: sudo apt install -y protobuf-compiler ``` -### Install fontconfig-dev library +### Install SDL2 library -You will need to install the fontconfig-dev library. This can be done by executing: +You will need to install the libsdl2-dev library. This can be done by executing: ```shell -sudo apt install -y libfontconfig-dev +sudo apt install -y libsdl2-dev ``` ### Install MQTT Broker diff --git a/samples/common/Cargo.toml b/samples/common/Cargo.toml index 32bcda56..ac2b9ac2 100644 --- a/samples/common/Cargo.toml +++ b/samples/common/Cargo.toml @@ -10,8 +10,10 @@ license = "MIT" [dependencies] config = { workspace = true } +image = { workspace = true } log = { workspace = true } samples-protobuf-data-access = { path = "../protobuf_data_access" } +sdl2 = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_derive = { workspace = true } tokio = { workspace = true } diff --git a/samples/common/src/image_rendering.rs b/samples/common/src/image_rendering.rs new file mode 100644 index 00000000..4ad32d59 --- /dev/null +++ b/samples/common/src/image_rendering.rs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +use image::{imageops::FilterType, DynamicImage}; +use sdl2::pixels::{Color, PixelFormatEnum}; +use sdl2::rect::Rect; +use sdl2::render::WindowCanvas; +use sdl2::surface::Surface; +use sdl2::Sdl; + +// This module is based on SDL2 (Simple DirectMedia Layer). It complements the sdl2 crate by providing +// methods that make it easier to render images. + +/// Create a canvas with an enclosing window. +/// +/// # Arguments +/// * `sdl_context` - The SDL context. +/// * `window_title` - The window's title. +/// * `window_width` - The window's width. +/// * `window_height` - The window's height. +pub fn create_canvas( + sdl_context: &mut Sdl, + window_title: &str, + window_width: u32, + window_height: u32, +) -> Result { + let video_subsystem = sdl_context.video()?; + + let window = video_subsystem + .window(window_title, window_width, window_height) + .position_centered() + .allow_highdpi() + .build() + .map_err(|err| format!("{}", err))?; + + let mut canvas = window.into_canvas().build().map_err(|err| format!("{}", err))?; + + // Set the background color to black. + canvas.set_draw_color(Color::RGB(0, 0, 0)); + + Ok(canvas) +} + +/// Resize an image to fit inside a canvas. +/// +/// # Arguments +/// * `image` - The image that needs to be resized. +/// * `canvas` - The canvas that it needs to fit in. +pub fn resize_image_to_fit_in_canvas( + image: DynamicImage, + canvas: &WindowCanvas, +) -> Result { + let (window_width, window_height): (u32, u32) = canvas.output_size()?; + + let width_scale = window_width as f32 / image.width() as f32; + let height_scale = window_height as f32 / image.height() as f32; + let scale: f32 = height_scale.min(width_scale); + + let resized_image_width = (scale * image.width() as f32) as u32; + let resized_image_height = (scale * image.height() as f32) as u32; + + Ok(image.resize(resized_image_width, resized_image_height, FilterType::Triangle)) +} + +/// Render an image to a canvas. +/// +/// # Arguments +/// * `image` - The image that we want to render. +/// * `canvas` - The canvas that will render the image. +pub fn render_image_to_canvas( + image: &DynamicImage, + canvas: &mut WindowCanvas, +) -> Result<(), String> { + // Prepare the image for copying it to a surface. + let rgb_image = image.to_rgb8(); + let mut image_buffer = rgb_image.into_raw(); + + let image_width = image.width(); + let image_height = image.height(); + // The pitch is the width of the texture times the size of a single pixel in bytes. + // Since we are using 24 bit pixels (RGB24), we need to mutiple the width by 3. + let image_pitch: u32 = image_width * 3; + + let surface = Surface::from_data( + &mut image_buffer, + image_width, + image_height, + image_pitch, + PixelFormatEnum::RGB24, + ) + .map_err(|err| err.to_string())?; + + let texture_creator = canvas.texture_creator(); + let texture = texture_creator + .create_texture_from_surface(surface) + .map_err(|err| format!("Failed to create texture from surface due to: {err}"))?; + + // Render the image. + canvas.clear(); + canvas.copy(&texture, None, Rect::new(0, 0, image_width, image_height))?; + canvas.present(); + + Ok(()) +} diff --git a/samples/common/src/lib.rs b/samples/common/src/lib.rs index 27524aae..43ca95c0 100644 --- a/samples/common/src/lib.rs +++ b/samples/common/src/lib.rs @@ -4,5 +4,6 @@ pub mod constants; pub mod consumer_config; +pub mod image_rendering; pub mod provider_config; pub mod utils; diff --git a/samples/streaming/Cargo.toml b/samples/streaming/Cargo.toml index 70a201a5..1a1b8b41 100644 --- a/samples/streaming/Cargo.toml +++ b/samples/streaming/Cargo.toml @@ -20,10 +20,10 @@ parking_lot = { workspace = true } prost = { workspace = true } samples-common = { path = "../common" } samples-protobuf-data-access = { path = "../protobuf_data_access" } +sdl2 = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_derive = { workspace = true } serde_json = { workspace = true } -show-image = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal", "sync"] } tokio-stream = { workspace = true } tonic = { workspace = true } diff --git a/samples/streaming/consumer/src/main.rs b/samples/streaming/consumer/src/main.rs index af1b12e5..9eb1bfbf 100644 --- a/samples/streaming/consumer/src/main.rs +++ b/samples/streaming/consumer/src/main.rs @@ -6,16 +6,15 @@ mod streaming_consumer_config; use digital_twin_model::sdv_v1 as sdv; use env_logger::{Builder, Target}; - -use image::io::Reader as ImageReader; +use image::{DynamicImage, io::Reader as ImageReader}; use log::{info, LevelFilter, warn}; use samples_common::constants::{digital_twin_operation, digital_twin_protocol}; +use samples_common::image_rendering::{create_canvas, render_image_to_canvas, resize_image_to_fit_in_canvas}; use samples_common::utils::{ discover_digital_twin_provider_using_ibeji, retrieve_invehicle_digital_twin_uri, }; use samples_protobuf_data_access::sample_grpc::v1::digital_twin_provider::StreamRequest; use samples_protobuf_data_access::sample_grpc::v1::digital_twin_provider::digital_twin_provider_client::DigitalTwinProviderClient; -use show_image::{ImageView, ImageInfo, create_window, WindowProxy}; use std::error::Error; use std::io::Cursor; use tokio_stream::StreamExt; @@ -25,14 +24,17 @@ use tonic::transport::Channel; /// /// # Arguments /// * `client` - The client connection to the service that will transfer the stream. +/// * `entity_id` - The entity id that is to be streamed. /// * `number_of_images` - The number of images that we will stream. -/// * `window` - The window where the streamed images will be shown. async fn stream_images( client: &mut DigitalTwinProviderClient, entity_id: &str, number_of_images: usize, - window: &mut WindowProxy, ) -> Result<(), Box> { + let mut sdl_context = sdl2::init()?; + + let mut canvas = create_canvas(&mut sdl_context, "Streamed Image", 800, 500)?; + let stream = client.stream(StreamRequest { entity_id: entity_id.to_string() }).await?.into_inner(); @@ -46,11 +48,20 @@ async fn stream_images( } let media_content = opt_media.unwrap().media_content; let image_reader = ImageReader::new(Cursor::new(media_content)).with_guessed_format()?; - let image = image_reader.decode()?; - let image_data = image.as_bytes().to_vec(); - let image_view = - ImageView::new(ImageInfo::rgb8(image.width(), image.height()), &image_data); - window.set_image("some file", image_view)?; + let image: DynamicImage = image_reader.decode()?; + + let resized_image = match resize_image_to_fit_in_canvas(image, &canvas) { + Ok(value) => value, + Err(err) => { + warn!("Failed to resize the image due to: {err}"); + // Skip this image. + continue; + } + }; + + if let Err(err) = render_image_to_canvas(&resized_image, &mut canvas) { + warn!("Failed to render the image due to: {err}"); + } } // The stream is dropped when we exit the function and the disconnect info is sent to the server. @@ -59,7 +70,6 @@ async fn stream_images( } #[tokio::main] -#[show_image::main] async fn main() -> Result<(), Box> { // Setup logging. Builder::new().filter(None, LevelFilter::Info).target(Target::Stdout).init(); @@ -87,17 +97,8 @@ async fn main() -> Result<(), Box> { let provider_uri = provider_endpoint_info.uri; info!("The provider URI for the Cabin Camera Feed property's provider is {provider_uri}"); - // Create a window with default options and display the image. - let mut window = create_window("image", Default::default())?; - let mut client = DigitalTwinProviderClient::connect(provider_uri.clone()).await.unwrap(); - stream_images( - &mut client, - sdv::camera::feed::ID, - settings.number_of_images.into(), - &mut window, - ) - .await?; + stream_images(&mut client, sdv::camera::feed::ID, settings.number_of_images.into()).await?; info!("The Consumer has completed.");