diff --git a/README.md b/README.md
index 7488e90d..05ee2baf 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,7 @@ HoloLens 2 server software and Python client library for streaming sensor data v
- Download calibration data (e.g., camera intrinsics, extrinsics, undistort maps) for the Front Camera and Research Mode sensors (except RM IMU Magnetometer).
- Optional per-frame pose for the Front Camera and Research Mode sensors.
+- Support for Mixed Reality Capture (Holograms in Front Camera video).
- Client can configure the bitrate and properties of the [H264](https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-encoder), [HEVC](https://learn.microsoft.com/en-us/windows/win32/medfound/h-265---hevc-video-encoder), and [AAC](https://learn.microsoft.com/en-us/windows/win32/medfound/aac-encoder) encoded streams.
- Client can configure the resolution and framerate of the Front Camera. See [here](https://github.com/jdibenes/hl2ss/blob/main/etc/pv_configurations.txt) for a list of supported configurations.
- Client can configure the focus, white balance, and exposure of the Front Camera. See [here](https://github.com/jdibenes/hl2ss/blob/main/viewer/client_ipc_rc.py).
diff --git a/hl2ss/hl2ss/Package.appxmanifest b/hl2ss/hl2ss/Package.appxmanifest
index 03a84b7a..600a3b15 100644
--- a/hl2ss/hl2ss/Package.appxmanifest
+++ b/hl2ss/hl2ss/Package.appxmanifest
@@ -9,7 +9,7 @@
+ Version="1.0.24.0" />
hl2ss
diff --git a/hl2ss/hl2ss/custom_video_effect.cpp b/hl2ss/hl2ss/custom_video_effect.cpp
new file mode 100644
index 00000000..8ce7e08f
--- /dev/null
+++ b/hl2ss/hl2ss/custom_video_effect.cpp
@@ -0,0 +1,30 @@
+
+#include "custom_video_effect.h"
+
+using namespace winrt::Windows::Foundation::Collections;
+using namespace winrt::Windows::Media::Effects;
+
+MRCVideoEffect::MRCVideoEffect(MRCOptions const& options)
+{
+ m_propertySet.Insert(L"StreamType", winrt::box_value(1));
+ m_propertySet.Insert(L"HologramCompositionEnabled", winrt::box_value(options.hologram_composition));
+ m_propertySet.Insert(L"RecordingIndicatorEnabled", winrt::box_value(options.recording_indicator));
+ m_propertySet.Insert(L"VideoStabilizationEnabled", winrt::box_value(options.video_stabilization));
+ m_propertySet.Insert(L"VideoStabilizationBufferLength", winrt::box_value(options.video_stabilization_length));
+ m_propertySet.Insert(L"GlobalOpacityCoefficient", winrt::box_value(options.global_opacity));
+ m_propertySet.Insert(L"BlankOnProtectedContent", winrt::box_value(options.blank_protected));
+ m_propertySet.Insert(L"ShowHiddenMesh", winrt::box_value(options.show_mesh));
+ m_propertySet.Insert(L"OutputSize", winrt::box_value(winrt::Windows::Foundation::Size(options.output_width, options.output_height)));
+ m_propertySet.Insert(L"OutputSubtype", winrt::box_value(winrt::hstring(L"Nv12")));
+ m_propertySet.Insert(L"PreferredHologramPerspective", winrt::box_value(options.hologram_perspective));
+}
+
+IPropertySet MRCVideoEffect::Properties()
+{
+ return m_propertySet;
+}
+
+winrt::hstring MRCVideoEffect::ActivatableClassId()
+{
+ return L"Windows.Media.MixedRealityCapture.MixedRealityCaptureVideoEffect";
+}
diff --git a/hl2ss/hl2ss/custom_video_effect.h b/hl2ss/hl2ss/custom_video_effect.h
new file mode 100644
index 00000000..9ae54962
--- /dev/null
+++ b/hl2ss/hl2ss/custom_video_effect.h
@@ -0,0 +1,32 @@
+
+#pragma once
+
+#include
+#include
+#include
+
+struct MRCOptions
+{
+ bool enable;
+ bool hologram_composition;
+ bool recording_indicator;
+ bool video_stabilization;
+ bool blank_protected;
+ bool show_mesh;
+ uint8_t _reserved[2];
+ float global_opacity;
+ float output_width;
+ float output_height;
+ uint32_t video_stabilization_length;
+ uint32_t hologram_perspective;
+};
+
+struct MRCVideoEffect : winrt::implements
+{
+ winrt::Windows::Foundation::Collections::PropertySet m_propertySet;
+
+ MRCVideoEffect(MRCOptions const& options);
+
+ winrt::hstring ActivatableClassId();
+ winrt::Windows::Foundation::Collections::IPropertySet Properties();
+};
diff --git a/hl2ss/hl2ss/hl2ss.vcxproj b/hl2ss/hl2ss/hl2ss.vcxproj
index 6a2d9c17..6dba185d 100644
--- a/hl2ss/hl2ss/hl2ss.vcxproj
+++ b/hl2ss/hl2ss/hl2ss.vcxproj
@@ -190,6 +190,7 @@
+
@@ -264,6 +265,7 @@
+
diff --git a/hl2ss/hl2ss/hl2ss.vcxproj.filters b/hl2ss/hl2ss/hl2ss.vcxproj.filters
index 9f1d27fc..7f617ec8 100644
--- a/hl2ss/hl2ss/hl2ss.vcxproj.filters
+++ b/hl2ss/hl2ss/hl2ss.vcxproj.filters
@@ -217,6 +217,9 @@
Libraries\Zdepth\zstd
+
+ Source Files\sink
+
@@ -561,6 +564,9 @@
Libraries\Zdepth\zstd
+
+ Header Files\sink
+
diff --git a/hl2ss/hl2ss/ipc_sc.cpp b/hl2ss/hl2ss/ipc_sc.cpp
index aaf11711..4065fcf6 100644
--- a/hl2ss/hl2ss/ipc_sc.cpp
+++ b/hl2ss/hl2ss/ipc_sc.cpp
@@ -2,6 +2,7 @@
#include "server.h"
#include "custom_media_types.h"
#include "custom_sink_writers.h"
+#include "custom_video_effect.h"
#include "ipc_sc.h"
#include
@@ -168,3 +169,34 @@ bool ReceiveZABFormat_Profile(SOCKET clientsocket, ZABFormat& format)
return true;
}
+
+// OK
+bool ReceiveMRCOptions(SOCKET clientsocket, MRCOptions& options)
+{
+ bool ok;
+
+ ok = recv_u8(clientsocket, *(uint8_t*)&options.enable);
+ if (!ok) { return false; }
+ ok = recv_u8(clientsocket, *(uint8_t*)&options.hologram_composition);
+ if (!ok) { return false; }
+ ok = recv_u8(clientsocket, *(uint8_t*)&options.recording_indicator);
+ if (!ok) { return false; }
+ ok = recv_u8(clientsocket, *(uint8_t*)&options.video_stabilization);
+ if (!ok) { return false; }
+ ok = recv_u8(clientsocket, *(uint8_t*)&options.blank_protected);
+ if (!ok) { return false; }
+ ok = recv_u8(clientsocket, *(uint8_t*)&options.show_mesh);
+ if (!ok) { return false; }
+ ok = recv_u32(clientsocket, *(uint32_t*)&options.global_opacity);
+ if (!ok) { return false; }
+ ok = recv_u32(clientsocket, *(uint32_t*)&options.output_width);
+ if (!ok) { return false; }
+ ok = recv_u32(clientsocket, *(uint32_t*)&options.output_height);
+ if (!ok) { return false; }
+ ok = recv_u32(clientsocket, options.video_stabilization_length);
+ if (!ok) { return false; }
+ ok = recv_u32(clientsocket, options.hologram_perspective);
+ if (!ok) { return false; }
+
+ return true;
+}
diff --git a/hl2ss/hl2ss/ipc_sc.h b/hl2ss/hl2ss/ipc_sc.h
index 96acceed..a2a50d9e 100644
--- a/hl2ss/hl2ss/ipc_sc.h
+++ b/hl2ss/hl2ss/ipc_sc.h
@@ -4,6 +4,7 @@
#include "server.h"
#include "custom_media_types.h"
#include "custom_sink_writers.h"
+#include "custom_video_effect.h"
#include
@@ -27,3 +28,4 @@ bool ReceiveH26xFormat_Profile(SOCKET clientsocket, H26xFormat& format);
bool ReceiveH26xEncoder_Options(SOCKET clientsocket, std::vector& options);
bool ReceiveZABFormat_PNGFilter(SOCKET clientsocket, ZABFormat& format);
bool ReceiveZABFormat_Profile(SOCKET clientsocket, ZABFormat& format);
+bool ReceiveMRCOptions(SOCKET clientsocket, MRCOptions& options);
diff --git a/hl2ss/hl2ss/personal_video.cpp b/hl2ss/hl2ss/personal_video.cpp
index b3fb51f3..39821b0f 100644
--- a/hl2ss/hl2ss/personal_video.cpp
+++ b/hl2ss/hl2ss/personal_video.cpp
@@ -1,5 +1,6 @@
#include "lock.h"
+#include "custom_video_effect.h"
#include
#include
@@ -93,7 +94,7 @@ void PersonalVideo_Cleanup()
}
// OK
-void PersonalVideo_Open()
+void PersonalVideo_Open(MRCOptions const& options)
{
uint32_t const width = 1920;
uint32_t const height = 1080;
@@ -121,6 +122,8 @@ void PersonalVideo_Open()
g_mediaCapture.InitializeAsync(settings).get();
+ if (options.enable) { g_mediaCapture.AddVideoEffectAsync(MRCVideoEffect(options), MediaStreamType::VideoRecord).get(); }
+
PersonalVideo_FindVideoSource(g_mediaCapture, g_videoSource);
g_ready = true;
diff --git a/hl2ss/hl2ss/personal_video.h b/hl2ss/hl2ss/personal_video.h
index 443a744f..17132a23 100644
--- a/hl2ss/hl2ss/personal_video.h
+++ b/hl2ss/hl2ss/personal_video.h
@@ -1,11 +1,12 @@
#pragma once
+#include "custom_video_effect.h"
#include
void PersonalVideo_Initialize();
void PersonalVideo_Cleanup();
-void PersonalVideo_Open();
+void PersonalVideo_Open(MRCOptions const& options);
void PersonalVideo_Close();
bool PersonalVideo_Status();
bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate);
diff --git a/hl2ss/hl2ss/stream_pv.cpp b/hl2ss/hl2ss/stream_pv.cpp
index 507e0936..86e232f5 100644
--- a/hl2ss/hl2ss/stream_pv.cpp
+++ b/hl2ss/hl2ss/stream_pv.cpp
@@ -251,12 +251,19 @@ static void PV_Stream(SOCKET clientsocket)
ok = recv_u8(clientsocket, mode);
if (!ok) { return; }
- if (!PersonalVideo_Status() && (mode & 4)) { PersonalVideo_Open(); }
- if (!PersonalVideo_Status()) { return; }
-
ok = ReceiveH26xFormat_Video(clientsocket, format);
if (!ok) { return; }
+ if (mode & 4)
+ {
+ MRCOptions options;
+ ok = ReceiveMRCOptions(clientsocket, options);
+ if (!ok) { return; }
+ if (!PersonalVideo_Status()) { PersonalVideo_Open(options); }
+ }
+
+ if (!PersonalVideo_Status()) { return; }
+
ok = PersonalVideo_SetFormat(format.width, format.height, format.framerate);
if (!ok) { return; }
diff --git a/hl2ss/plugin/plugin.vcxproj b/hl2ss/plugin/plugin.vcxproj
index 76030784..c4992ab0 100644
--- a/hl2ss/plugin/plugin.vcxproj
+++ b/hl2ss/plugin/plugin.vcxproj
@@ -281,6 +281,7 @@
+
@@ -352,6 +353,7 @@
+
diff --git a/hl2ss/plugin/plugin.vcxproj.filters b/hl2ss/plugin/plugin.vcxproj.filters
index 3e8112c9..388e191b 100644
--- a/hl2ss/plugin/plugin.vcxproj.filters
+++ b/hl2ss/plugin/plugin.vcxproj.filters
@@ -219,6 +219,9 @@
Libraries\Zdepth
+
+ Libraries\hl2ss
+
@@ -434,6 +437,9 @@
Libraries\Zdepth
+
+ Libraries\hl2ss
+
diff --git a/unity/Assets/Plugins/WSA/hl2ss.dll b/unity/Assets/Plugins/WSA/hl2ss.dll
index a3c967ae..7a2ab954 100644
Binary files a/unity/Assets/Plugins/WSA/hl2ss.dll and b/unity/Assets/Plugins/WSA/hl2ss.dll differ
diff --git a/viewer/client_stream_pv.py b/viewer/client_stream_pv.py
index 3a80da4d..e048d9c2 100644
--- a/viewer/client_stream_pv.py
+++ b/viewer/client_stream_pv.py
@@ -26,6 +26,9 @@
# 2: query calibration (single transfer)
mode = hl2ss.StreamMode.MODE_1
+# Enable Mixed Reality Capture (Holograms)
+enable_mrc = False
+
# Camera parameters
width = 1920
height = 1080
@@ -49,7 +52,7 @@
#------------------------------------------------------------------------------
-hl2ss_lnm.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO)
+hl2ss_lnm.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, enable_mrc=enable_mrc)
if (mode == hl2ss.StreamMode.MODE_2):
data = hl2ss_lnm.download_calibration_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, width, height, framerate)
diff --git a/viewer/hl2ss.py b/viewer/hl2ss.py
index 0e4b9232..fc5fa453 100644
--- a/viewer/hl2ss.py
+++ b/viewer/hl2ss.py
@@ -161,6 +161,11 @@ class PNGFilterMode:
ADAPTIVE = 6
+class HologramPerspective:
+ DISPLAY = 0
+ PV = 1
+
+
# RM VLC Parameters
class Parameters_RM_VLC:
WIDTH = 640
@@ -440,7 +445,11 @@ def _create_configuration_for_h26x_encoding(options):
for key, value in options.items():
configuration.extend(struct.pack('