diff --git a/README.md b/README.md index 7488e90d..05ee2baf 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ HoloLens 2 server software and Python client library for streaming sensor data v - Download calibration data (e.g., camera intrinsics, extrinsics, undistort maps) for the Front Camera and Research Mode sensors (except RM IMU Magnetometer). - Optional per-frame pose for the Front Camera and Research Mode sensors. +- Support for Mixed Reality Capture (Holograms in Front Camera video). - Client can configure the bitrate and properties of the [H264](https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-encoder), [HEVC](https://learn.microsoft.com/en-us/windows/win32/medfound/h-265---hevc-video-encoder), and [AAC](https://learn.microsoft.com/en-us/windows/win32/medfound/aac-encoder) encoded streams. - Client can configure the resolution and framerate of the Front Camera. See [here](https://github.com/jdibenes/hl2ss/blob/main/etc/pv_configurations.txt) for a list of supported configurations. - Client can configure the focus, white balance, and exposure of the Front Camera. See [here](https://github.com/jdibenes/hl2ss/blob/main/viewer/client_ipc_rc.py). diff --git a/hl2ss/hl2ss/Package.appxmanifest b/hl2ss/hl2ss/Package.appxmanifest index 03a84b7a..600a3b15 100644 --- a/hl2ss/hl2ss/Package.appxmanifest +++ b/hl2ss/hl2ss/Package.appxmanifest @@ -9,7 +9,7 @@ + Version="1.0.24.0" /> hl2ss diff --git a/hl2ss/hl2ss/custom_video_effect.cpp b/hl2ss/hl2ss/custom_video_effect.cpp new file mode 100644 index 00000000..8ce7e08f --- /dev/null +++ b/hl2ss/hl2ss/custom_video_effect.cpp @@ -0,0 +1,30 @@ + +#include "custom_video_effect.h" + +using namespace winrt::Windows::Foundation::Collections; +using namespace winrt::Windows::Media::Effects; + +MRCVideoEffect::MRCVideoEffect(MRCOptions const& options) +{ + m_propertySet.Insert(L"StreamType", winrt::box_value(1)); + m_propertySet.Insert(L"HologramCompositionEnabled", winrt::box_value(options.hologram_composition)); + m_propertySet.Insert(L"RecordingIndicatorEnabled", winrt::box_value(options.recording_indicator)); + m_propertySet.Insert(L"VideoStabilizationEnabled", winrt::box_value(options.video_stabilization)); + m_propertySet.Insert(L"VideoStabilizationBufferLength", winrt::box_value(options.video_stabilization_length)); + m_propertySet.Insert(L"GlobalOpacityCoefficient", winrt::box_value(options.global_opacity)); + m_propertySet.Insert(L"BlankOnProtectedContent", winrt::box_value(options.blank_protected)); + m_propertySet.Insert(L"ShowHiddenMesh", winrt::box_value(options.show_mesh)); + m_propertySet.Insert(L"OutputSize", winrt::box_value(winrt::Windows::Foundation::Size(options.output_width, options.output_height))); + m_propertySet.Insert(L"OutputSubtype", winrt::box_value(winrt::hstring(L"Nv12"))); + m_propertySet.Insert(L"PreferredHologramPerspective", winrt::box_value(options.hologram_perspective)); +} + +IPropertySet MRCVideoEffect::Properties() +{ + return m_propertySet; +} + +winrt::hstring MRCVideoEffect::ActivatableClassId() +{ + return L"Windows.Media.MixedRealityCapture.MixedRealityCaptureVideoEffect"; +} diff --git a/hl2ss/hl2ss/custom_video_effect.h b/hl2ss/hl2ss/custom_video_effect.h new file mode 100644 index 00000000..9ae54962 --- /dev/null +++ b/hl2ss/hl2ss/custom_video_effect.h @@ -0,0 +1,32 @@ + +#pragma once + +#include +#include +#include + +struct MRCOptions +{ + bool enable; + bool hologram_composition; + bool recording_indicator; + bool video_stabilization; + bool blank_protected; + bool show_mesh; + uint8_t _reserved[2]; + float global_opacity; + float output_width; + float output_height; + uint32_t video_stabilization_length; + uint32_t hologram_perspective; +}; + +struct MRCVideoEffect : winrt::implements +{ + winrt::Windows::Foundation::Collections::PropertySet m_propertySet; + + MRCVideoEffect(MRCOptions const& options); + + winrt::hstring ActivatableClassId(); + winrt::Windows::Foundation::Collections::IPropertySet Properties(); +}; diff --git a/hl2ss/hl2ss/hl2ss.vcxproj b/hl2ss/hl2ss/hl2ss.vcxproj index 6a2d9c17..6dba185d 100644 --- a/hl2ss/hl2ss/hl2ss.vcxproj +++ b/hl2ss/hl2ss/hl2ss.vcxproj @@ -190,6 +190,7 @@ + @@ -264,6 +265,7 @@ + diff --git a/hl2ss/hl2ss/hl2ss.vcxproj.filters b/hl2ss/hl2ss/hl2ss.vcxproj.filters index 9f1d27fc..7f617ec8 100644 --- a/hl2ss/hl2ss/hl2ss.vcxproj.filters +++ b/hl2ss/hl2ss/hl2ss.vcxproj.filters @@ -217,6 +217,9 @@ Libraries\Zdepth\zstd + + Source Files\sink + @@ -561,6 +564,9 @@ Libraries\Zdepth\zstd + + Header Files\sink + diff --git a/hl2ss/hl2ss/ipc_sc.cpp b/hl2ss/hl2ss/ipc_sc.cpp index aaf11711..4065fcf6 100644 --- a/hl2ss/hl2ss/ipc_sc.cpp +++ b/hl2ss/hl2ss/ipc_sc.cpp @@ -2,6 +2,7 @@ #include "server.h" #include "custom_media_types.h" #include "custom_sink_writers.h" +#include "custom_video_effect.h" #include "ipc_sc.h" #include @@ -168,3 +169,34 @@ bool ReceiveZABFormat_Profile(SOCKET clientsocket, ZABFormat& format) return true; } + +// OK +bool ReceiveMRCOptions(SOCKET clientsocket, MRCOptions& options) +{ + bool ok; + + ok = recv_u8(clientsocket, *(uint8_t*)&options.enable); + if (!ok) { return false; } + ok = recv_u8(clientsocket, *(uint8_t*)&options.hologram_composition); + if (!ok) { return false; } + ok = recv_u8(clientsocket, *(uint8_t*)&options.recording_indicator); + if (!ok) { return false; } + ok = recv_u8(clientsocket, *(uint8_t*)&options.video_stabilization); + if (!ok) { return false; } + ok = recv_u8(clientsocket, *(uint8_t*)&options.blank_protected); + if (!ok) { return false; } + ok = recv_u8(clientsocket, *(uint8_t*)&options.show_mesh); + if (!ok) { return false; } + ok = recv_u32(clientsocket, *(uint32_t*)&options.global_opacity); + if (!ok) { return false; } + ok = recv_u32(clientsocket, *(uint32_t*)&options.output_width); + if (!ok) { return false; } + ok = recv_u32(clientsocket, *(uint32_t*)&options.output_height); + if (!ok) { return false; } + ok = recv_u32(clientsocket, options.video_stabilization_length); + if (!ok) { return false; } + ok = recv_u32(clientsocket, options.hologram_perspective); + if (!ok) { return false; } + + return true; +} diff --git a/hl2ss/hl2ss/ipc_sc.h b/hl2ss/hl2ss/ipc_sc.h index 96acceed..a2a50d9e 100644 --- a/hl2ss/hl2ss/ipc_sc.h +++ b/hl2ss/hl2ss/ipc_sc.h @@ -4,6 +4,7 @@ #include "server.h" #include "custom_media_types.h" #include "custom_sink_writers.h" +#include "custom_video_effect.h" #include @@ -27,3 +28,4 @@ bool ReceiveH26xFormat_Profile(SOCKET clientsocket, H26xFormat& format); bool ReceiveH26xEncoder_Options(SOCKET clientsocket, std::vector& options); bool ReceiveZABFormat_PNGFilter(SOCKET clientsocket, ZABFormat& format); bool ReceiveZABFormat_Profile(SOCKET clientsocket, ZABFormat& format); +bool ReceiveMRCOptions(SOCKET clientsocket, MRCOptions& options); diff --git a/hl2ss/hl2ss/personal_video.cpp b/hl2ss/hl2ss/personal_video.cpp index b3fb51f3..39821b0f 100644 --- a/hl2ss/hl2ss/personal_video.cpp +++ b/hl2ss/hl2ss/personal_video.cpp @@ -1,5 +1,6 @@ #include "lock.h" +#include "custom_video_effect.h" #include #include @@ -93,7 +94,7 @@ void PersonalVideo_Cleanup() } // OK -void PersonalVideo_Open() +void PersonalVideo_Open(MRCOptions const& options) { uint32_t const width = 1920; uint32_t const height = 1080; @@ -121,6 +122,8 @@ void PersonalVideo_Open() g_mediaCapture.InitializeAsync(settings).get(); + if (options.enable) { g_mediaCapture.AddVideoEffectAsync(MRCVideoEffect(options), MediaStreamType::VideoRecord).get(); } + PersonalVideo_FindVideoSource(g_mediaCapture, g_videoSource); g_ready = true; diff --git a/hl2ss/hl2ss/personal_video.h b/hl2ss/hl2ss/personal_video.h index 443a744f..17132a23 100644 --- a/hl2ss/hl2ss/personal_video.h +++ b/hl2ss/hl2ss/personal_video.h @@ -1,11 +1,12 @@ #pragma once +#include "custom_video_effect.h" #include void PersonalVideo_Initialize(); void PersonalVideo_Cleanup(); -void PersonalVideo_Open(); +void PersonalVideo_Open(MRCOptions const& options); void PersonalVideo_Close(); bool PersonalVideo_Status(); bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate); diff --git a/hl2ss/hl2ss/stream_pv.cpp b/hl2ss/hl2ss/stream_pv.cpp index 507e0936..86e232f5 100644 --- a/hl2ss/hl2ss/stream_pv.cpp +++ b/hl2ss/hl2ss/stream_pv.cpp @@ -251,12 +251,19 @@ static void PV_Stream(SOCKET clientsocket) ok = recv_u8(clientsocket, mode); if (!ok) { return; } - if (!PersonalVideo_Status() && (mode & 4)) { PersonalVideo_Open(); } - if (!PersonalVideo_Status()) { return; } - ok = ReceiveH26xFormat_Video(clientsocket, format); if (!ok) { return; } + if (mode & 4) + { + MRCOptions options; + ok = ReceiveMRCOptions(clientsocket, options); + if (!ok) { return; } + if (!PersonalVideo_Status()) { PersonalVideo_Open(options); } + } + + if (!PersonalVideo_Status()) { return; } + ok = PersonalVideo_SetFormat(format.width, format.height, format.framerate); if (!ok) { return; } diff --git a/hl2ss/plugin/plugin.vcxproj b/hl2ss/plugin/plugin.vcxproj index 76030784..c4992ab0 100644 --- a/hl2ss/plugin/plugin.vcxproj +++ b/hl2ss/plugin/plugin.vcxproj @@ -281,6 +281,7 @@ + @@ -352,6 +353,7 @@ + diff --git a/hl2ss/plugin/plugin.vcxproj.filters b/hl2ss/plugin/plugin.vcxproj.filters index 3e8112c9..388e191b 100644 --- a/hl2ss/plugin/plugin.vcxproj.filters +++ b/hl2ss/plugin/plugin.vcxproj.filters @@ -219,6 +219,9 @@ Libraries\Zdepth + + Libraries\hl2ss + @@ -434,6 +437,9 @@ Libraries\Zdepth + + Libraries\hl2ss + diff --git a/unity/Assets/Plugins/WSA/hl2ss.dll b/unity/Assets/Plugins/WSA/hl2ss.dll index a3c967ae..7a2ab954 100644 Binary files a/unity/Assets/Plugins/WSA/hl2ss.dll and b/unity/Assets/Plugins/WSA/hl2ss.dll differ diff --git a/viewer/client_stream_pv.py b/viewer/client_stream_pv.py index 3a80da4d..e048d9c2 100644 --- a/viewer/client_stream_pv.py +++ b/viewer/client_stream_pv.py @@ -26,6 +26,9 @@ # 2: query calibration (single transfer) mode = hl2ss.StreamMode.MODE_1 +# Enable Mixed Reality Capture (Holograms) +enable_mrc = False + # Camera parameters width = 1920 height = 1080 @@ -49,7 +52,7 @@ #------------------------------------------------------------------------------ -hl2ss_lnm.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO) +hl2ss_lnm.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, enable_mrc=enable_mrc) if (mode == hl2ss.StreamMode.MODE_2): data = hl2ss_lnm.download_calibration_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, width, height, framerate) diff --git a/viewer/hl2ss.py b/viewer/hl2ss.py index 0e4b9232..fc5fa453 100644 --- a/viewer/hl2ss.py +++ b/viewer/hl2ss.py @@ -161,6 +161,11 @@ class PNGFilterMode: ADAPTIVE = 6 +class HologramPerspective: + DISPLAY = 0 + PV = 1 + + # RM VLC Parameters class Parameters_RM_VLC: WIDTH = 640 @@ -440,7 +445,11 @@ def _create_configuration_for_h26x_encoding(options): for key, value in options.items(): configuration.extend(struct.pack('