From 0f1413b49384ea666d7bc4f27fdca72c91b9e51a Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 5 Jul 2024 15:35:24 +0200 Subject: [PATCH 01/13] Added print of supported extensions --- src/devices/openxrheadset/OpenXrInterface.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index b6cb366..140f4a1 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -41,6 +41,8 @@ bool OpenXrInterface::checkExtensions() bool depth_supported = false; bool debug_supported = false; + std::stringstream supported_extensions; + supported_extensions << "Supported extensions: " <htc_trackers_supported = true; } + + supported_extensions << std::endl << " - " << ext_props[i].extensionName; } + yCInfo(OPENXRHEADSET) << supported_extensions.str(); + // A graphics extension like OpenGL is required to draw anything in VR if (!opengl_supported) { yCError(OPENXRHEADSET) << "Runtime does not support OpenGL extension!"; From bd3140c98a54d511f3ceeda16f9a30113d4e8d2e Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 8 Jul 2024 14:12:39 +0200 Subject: [PATCH 02/13] Added focus3 interaction profile --- src/devices/openxrheadset/OpenXrHeadset.h | 4 +- src/devices/openxrheadset/OpenXrInterface.cpp | 57 +++++++++++++++++++ .../impl/OpenXrInterfaceImpl.cpp | 5 ++ .../openxrheadset/impl/OpenXrInterfaceImpl.h | 4 ++ .../thrifts/OpenXrHeadsetCommands.thrift | 4 +- 5 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/devices/openxrheadset/OpenXrHeadset.h b/src/devices/openxrheadset/OpenXrHeadset.h index 5379aa4..4f23ebb 100644 --- a/src/devices/openxrheadset/OpenXrHeadset.h +++ b/src/devices/openxrheadset/OpenXrHeadset.h @@ -86,14 +86,14 @@ class yarp::dev::OpenXrHeadset : public yarp::dev::DeviceDriver, //OpenXrHeadsetCommands /** * Get the current interaction profile for the left hand - * It returns a string that can be one between none, khr_simple_controller, oculus_touch_controller or htc_vive_controller + * It returns a string that can be one between none, khr_simple_controller, oculus_touch_controller, htc_vive_controller, or htc_vive_focus3_controller * @return a string indicating the interaction profile in use. */ virtual std::string getLeftHandInteractionProfile() override; /** * Get the current interaction profile for the right hand - * It returns a string that can be one between none, khr_simple_controller, oculus_touch_controller or htc_vive_controller + * It returns a string that can be one between none, khr_simple_controller, oculus_touch_controller, htc_vive_controller, or htc_vive_focus3_controller * @return a string indicating the interaction profile in use. */ virtual std::string getRightHandInteractionProfile() override; diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index 140f4a1..01cb8f7 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -60,6 +60,10 @@ bool OpenXrInterface::checkExtensions() m_pimpl->htc_trackers_supported = true; } + if (strcmp(XR_HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_EXTENSION_NAME, ext_props[i].extensionName) == 0) { + m_pimpl->focus3_supported = true; + } + supported_extensions << std::endl << " - " << ext_props[i].extensionName; } @@ -85,6 +89,10 @@ bool OpenXrInterface::checkExtensions() yCWarning(OPENXRHEADSET) << "Runtime does not support the HTC Vive Trackers!"; } + if (!m_pimpl->focus3_supported) { + yCWarning(OPENXRHEADSET) << "Runtime does not support the HTC Vive Focus 3 controllers!"; + } + return true; } @@ -124,6 +132,10 @@ bool OpenXrInterface::prepareXrInstance() { requestedExtensions.push_back(XR_HTCX_VIVE_TRACKER_INTERACTION_EXTENSION_NAME); } + if (m_pimpl->focus3_supported) + { + requestedExtensions.push_back(XR_HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_EXTENSION_NAME); + } // Populate the info to create the instance XrInstanceCreateInfo instanceCreateInfo @@ -672,6 +684,51 @@ bool OpenXrInterface::prepareXrActions() right_hand.inputsDeclarations[HTC_VIVE_INTERACTION_PROFILE_TAG] = vive_left_inputs; //the inputs from the left and right hand are the same + if (m_pimpl->focus3_supported) + { + InputActionsDeclaration& focus3_left_inputs = left_hand.inputsDeclarations[HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_PROFILE_TAG]; + InputActionsDeclaration& focus3_right_inputs = right_hand.inputsDeclarations[HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_PROFILE_TAG]; + + focus3_left_inputs.poses = + { + {"/input/grip/pose", "grip"} + }; + focus3_right_inputs.poses = focus3_left_inputs.poses; + + + focus3_left_inputs.buttons = + { + //We avoid the menu button because it is used to open SteamVR when using the Streaming Hub + {"/input/x/click", "x"}, + {"/input/y/click", "y"}, + {"/input/trigger/click", "trigger_click"}, + {"/input/squeeze/click", "squeeze_click"}, + {"/input/thumbstick/click", "thumbstick_click"} + }; + focus3_right_inputs.buttons = + { + {"/input/a/click", "a"}, + {"/input/b/click", "b"}, + {"/input/trigger/click", "trigger_click"}, + {"/input/squeeze/click", "squeeze_click"}, + {"/input/thumbstick/click", "thumbstick_click"} + }; + + focus3_left_inputs.axes = + { + {"/input/trigger/value", "trigger"}, + {"/input/squeeze/value", "squeeze"} + }; + focus3_right_inputs.axes = focus3_left_inputs.axes; + + focus3_left_inputs.thumbsticks = + { + {"/input/thumbstick", "thumbstick"} + }; + + focus3_right_inputs.thumbsticks = focus3_left_inputs.thumbsticks; + } + std::vector topLevelPathsDeclaration = {left_hand, //The left hand should always come first in this list right_hand}; //The right hand should always come second in this list diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.cpp b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.cpp index 761db55..08b60b6 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.cpp +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.cpp @@ -407,6 +407,11 @@ std::string OpenXrInterface::Implementation::getInteractionProfileShortTag(const return "htc_vive_tracker"; } + if (interactionProfile == HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_PROFILE_TAG) + { + return "htc_vive_focus3_controller"; + } + return "none"; } diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h index 470f016..83f5fe9 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h @@ -34,6 +34,7 @@ #define OCULUS_TOUCH_INTERACTION_PROFILE_TAG "/interaction_profiles/oculus/touch_controller" #define HTC_VIVE_INTERACTION_PROFILE_TAG "/interaction_profiles/htc/vive_controller" #define HTC_VIVE_TRACKER_INTERACTION_PROFILE_TAG "/interaction_profiles/htc/vive_tracker_htcx" +#define HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_PROFILE_TAG "/interaction_profiles/htc/vive_focus3_controller" // STRUCTS template @@ -292,6 +293,9 @@ class OpenXrInterface::Implementation // Map defining which tracker is connected std::unordered_map htc_trackers_status; + // flag to check if the HTC VIVE Focus3 controllers are supported by the runtime. + bool focus3_supported = false; + // state of the application XrSessionState state = XR_SESSION_STATE_UNKNOWN; diff --git a/src/devices/openxrheadset/thrifts/OpenXrHeadsetCommands.thrift b/src/devices/openxrheadset/thrifts/OpenXrHeadsetCommands.thrift index 499fe51..49d256f 100644 --- a/src/devices/openxrheadset/thrifts/OpenXrHeadsetCommands.thrift +++ b/src/devices/openxrheadset/thrifts/OpenXrHeadsetCommands.thrift @@ -10,14 +10,14 @@ service OpenXrHeadsetCommands { /** * Get the current interaction profile for the left hand - * It returns a string that can be one between none, khr_simple_controller, oculus_touch_controller or htc_vive_controller + * It returns a string that can be one between none, khr_simple_controller, oculus_touch_controller, htc_vive_controller, or htc_vive_focus3_controller * @return a string indicating the interaction profile in use. */ string getLeftHandInteractionProfile(); /** * Get the current interaction profile for the right hand - * It returns a string that can be one between none, khr_simple_controller, oculus_touch_controller or htc_vive_controller + * It returns a string that can be one between none, khr_simple_controller, oculus_touch_controller, htc_vive_controller, or htc_vive_focus3_controller * @return a string indicating the interaction profile in use. */ string getRightHandInteractionProfile(); From a0bd5773f57a43dce16ec1840efc8de5a1e58205 Mon Sep 17 00:00:00 2001 From: Stefano Date: Tue, 10 Sep 2024 18:32:58 +0200 Subject: [PATCH 03/13] Added facial tracking extension --- src/devices/openxrheadset/OpenXrInterface.cpp | 141 +++++++++++++++++- src/devices/openxrheadset/OpenXrInterface.h | 2 +- .../openxrheadset/impl/OpenXrInterfaceImpl.h | 26 ++++ 3 files changed, 166 insertions(+), 3 deletions(-) diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index 01cb8f7..73c5c8d 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -64,6 +64,10 @@ bool OpenXrInterface::checkExtensions() m_pimpl->focus3_supported = true; } + if (strcmp(XR_HTC_FACIAL_TRACKING_EXTENSION_NAME, ext_props[i].extensionName) == 0) { + m_pimpl->htc_facial_tracking_extension_supported = true; + } + supported_extensions << std::endl << " - " << ext_props[i].extensionName; } @@ -93,6 +97,10 @@ bool OpenXrInterface::checkExtensions() yCWarning(OPENXRHEADSET) << "Runtime does not support the HTC Vive Focus 3 controllers!"; } + if (!m_pimpl->htc_facial_tracking_extension_supported) { + yCWarning(OPENXRHEADSET) << "Runtime does not support the HTC Vive Facial Tracking!"; + } + return true; } @@ -136,6 +144,10 @@ bool OpenXrInterface::prepareXrInstance() { requestedExtensions.push_back(XR_HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_EXTENSION_NAME); } + if (m_pimpl->htc_facial_tracking_extension_supported) + { + requestedExtensions.push_back(XR_HTC_FACIAL_TRACKING_EXTENSION_NAME); + } // Populate the info to create the instance XrInstanceCreateInfo instanceCreateInfo @@ -229,6 +241,24 @@ bool OpenXrInterface::prepareXrInstance() return false; } + if (m_pimpl->htc_facial_tracking_extension_supported) + { + result = xrGetInstanceProcAddr(m_pimpl->instance, "xrCreateFacialTrackerHTC", + (PFN_xrVoidFunction*)&(m_pimpl->pfn_xrCreateFacialTrackerHTC)); + if (!m_pimpl->checkXrOutput(result, "Failed to get the function to create the HTC facial tracker!")) + return false; + + result = xrGetInstanceProcAddr(m_pimpl->instance, "xrDestroyFacialTrackerHTC", + (PFN_xrVoidFunction*)&(m_pimpl->pfn_xrDestroyFacialTrackerHTC)); + if (!m_pimpl->checkXrOutput(result, "Failed to get the function to destroy the HTC facial tracker!")) + return false; + + result = xrGetInstanceProcAddr(m_pimpl->instance, "xrGetFacialExpressionsHTC", + (PFN_xrVoidFunction*)&(m_pimpl->pfn_xrGetFacialExpressionsHTC)); + if (!m_pimpl->checkXrOutput(result, "Failed to get the function to get the HTC facial expressions!")) + return false; + } + return true; } @@ -246,7 +276,7 @@ bool OpenXrInterface::prepareXrSystem() yCInfo(OPENXRHEADSET) << "Successfully got XrSystem with id" << m_pimpl->system_id << "for HMD form factor"; - printSystemProperties(); + checkSystemProperties(); uint32_t view_count = 0; // We first get the number of view configurations for the STEREO type @@ -317,12 +347,23 @@ bool OpenXrInterface::prepareXrSystem() return true; } -void OpenXrInterface::printSystemProperties() +void OpenXrInterface::checkSystemProperties() { XrSystemProperties system_props; system_props.type = XR_TYPE_SYSTEM_PROPERTIES; system_props.next = NULL; + XrSystemFacialTrackingPropertiesHTC facial_tracking_props; + facial_tracking_props.type = XR_TYPE_SYSTEM_FACIAL_TRACKING_PROPERTIES_HTC; + facial_tracking_props.next = NULL; + facial_tracking_props.supportEyeFacialTracking = XR_FALSE; + facial_tracking_props.supportLipFacialTracking = XR_FALSE; + + if (m_pimpl->htc_facial_tracking_extension_supported) + { + system_props.next = &facial_tracking_props; + } + XrResult result = xrGetSystemProperties(m_pimpl->instance, m_pimpl->system_id, &system_props); if (!XR_SUCCEEDED(result)) { @@ -339,6 +380,14 @@ void OpenXrInterface::printSystemProperties() system_props.graphicsProperties.maxSwapchainImageWidth); yCInfo(OPENXRHEADSET, "\tOrientation Tracking: %d", system_props.trackingProperties.orientationTracking); yCInfo(OPENXRHEADSET, "\tPosition Tracking : %d", system_props.trackingProperties.positionTracking); + + if (m_pimpl->htc_facial_tracking_extension_supported) + { + yCInfo(OPENXRHEADSET, "Facial tracking properties for system %lu: Eye tracking %d, Lip tracking %d", + system_props.systemId, facial_tracking_props.supportEyeFacialTracking, facial_tracking_props.supportLipFacialTracking); + m_pimpl->htc_eye_facial_tracking_supported = facial_tracking_props.supportEyeFacialTracking; + m_pimpl->htc_lip_facial_tracking_supported = facial_tracking_props.supportLipFacialTracking; + } } bool OpenXrInterface::prepareGL() @@ -435,6 +484,44 @@ bool OpenXrInterface::prepareXrSession() if (! m_pimpl->checkXrOutput(result, "Failed to create view space!")) return false; + if (m_pimpl->htc_eye_facial_tracking_supported) + { + XrFacialTrackerCreateInfoHTC facial_tracker_create_info = { + .type = XR_TYPE_FACIAL_TRACKER_CREATE_INFO_HTC, + .next = NULL, + .facialTrackingType = XR_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC, + }; + result = m_pimpl->pfn_xrCreateFacialTrackerHTC(m_pimpl->session, &facial_tracker_create_info, m_pimpl->htc_eye_facial_tracker); + if (!m_pimpl->checkXrOutput(result, "Failed to create eye facial tracker! Avoiding using it.")) + { + m_pimpl->htc_eye_facial_tracker = nullptr; + } + else + { + yCInfo(OPENXRHEADSET) << "Successfully created eye facial tracker!"; + m_pimpl->htc_eye_expressions.resize(XR_FACIAL_EXPRESSION_EYE_COUNT_HTC, 0.0); + } + } + + if (m_pimpl->htc_lip_facial_tracking_supported) + { + XrFacialTrackerCreateInfoHTC facial_tracker_create_info = { + .type = XR_TYPE_FACIAL_TRACKER_CREATE_INFO_HTC, + .next = NULL, + .facialTrackingType = XR_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC, + }; + result = m_pimpl->pfn_xrCreateFacialTrackerHTC(m_pimpl->session, &facial_tracker_create_info, m_pimpl->htc_lip_facial_tracker); + if (!m_pimpl->checkXrOutput(result, "Failed to create lip facial tracker! Avoiding using it.")) + { + m_pimpl->htc_lip_facial_tracker = nullptr; + } + else + { + yCInfo(OPENXRHEADSET) << "Successfully created lip facial tracker!"; + m_pimpl->htc_lip_expressions.resize(XR_FACIAL_EXPRESSION_LIP_COUNT_HTC, 0.0); + } + } + return true; } @@ -1056,6 +1143,44 @@ void OpenXrInterface::updateXrActions() m_pimpl->checkXrOutput(result, "Failed to update the status of %s!", thumbstick.name.c_str()); //Continue anyway } } + + if (m_pimpl->htc_eye_facial_tracker) + { + XrFacialExpressionsHTC expressions = { .type = XR_TYPE_FACIAL_EXPRESSIONS_HTC, + .next = NULL, + .isActive = XR_TRUE, + .sampleTime = m_pimpl->frame_state.predictedDisplayTime, + .expressionCount = static_cast(m_pimpl->htc_eye_expressions.size()), + .expressionWeightings = m_pimpl->htc_eye_expressions.data() }; + result = m_pimpl->pfn_xrGetFacialExpressionsHTC(*m_pimpl->htc_eye_facial_tracker, &expressions); + if (!m_pimpl->checkXrOutput(result, "Failed to get the facial expressions of the eye tracker!") || !expressions.isActive) + { + if (!expressions.isActive) + { + yCWarningThrottle(OPENXRHEADSET, 5.0, "The eye facial tracker is not active!"); + } + m_pimpl->htc_eye_expressions.assign(m_pimpl->htc_eye_expressions.size(), 0.0); + } + } + + if (m_pimpl->htc_lip_facial_tracker) + { + XrFacialExpressionsHTC expressions = { .type = XR_TYPE_FACIAL_EXPRESSIONS_HTC, + .next = NULL, + .isActive = XR_TRUE, + .sampleTime = m_pimpl->frame_state.predictedDisplayTime, + .expressionCount = static_cast(m_pimpl->htc_lip_expressions.size()), + .expressionWeightings = m_pimpl->htc_lip_expressions.data() }; + result = m_pimpl->pfn_xrGetFacialExpressionsHTC(*m_pimpl->htc_lip_facial_tracker, &expressions); + if (!m_pimpl->checkXrOutput(result, "Failed to get the facial expressions of the lip tracker!") || !expressions.isActive) + { + if (!expressions.isActive) + { + yCWarningThrottle(OPENXRHEADSET, 5.0, "The lip facial tracker is not active!"); + } + m_pimpl->htc_lip_expressions.assign(m_pimpl->htc_lip_expressions.size(), 0.0); + } + } } bool OpenXrInterface::updateInteractionProfiles() @@ -1824,6 +1949,18 @@ void OpenXrInterface::close() { m_pimpl->closing = true; + if (m_pimpl->htc_eye_facial_tracker) + { + m_pimpl->pfn_xrDestroyFacialTrackerHTC(*m_pimpl->htc_eye_facial_tracker); + m_pimpl->htc_eye_facial_tracker = nullptr; + } + + if (m_pimpl->htc_lip_facial_tracker) + { + m_pimpl->pfn_xrDestroyFacialTrackerHTC(*m_pimpl->htc_lip_facial_tracker); + m_pimpl->htc_lip_facial_tracker = nullptr; + } + if (m_pimpl->glFrameBufferId != 0) { glDeleteFramebuffers(1, &(m_pimpl->glFrameBufferId)); m_pimpl->glFrameBufferId = 0; diff --git a/src/devices/openxrheadset/OpenXrInterface.h b/src/devices/openxrheadset/OpenXrInterface.h index 00f1595..66b32ed 100644 --- a/src/devices/openxrheadset/OpenXrInterface.h +++ b/src/devices/openxrheadset/OpenXrInterface.h @@ -82,7 +82,7 @@ class OpenXrInterface bool prepareXrSystem(); - void printSystemProperties(); + void checkSystemProperties(); bool prepareGL(); diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h index 83f5fe9..a4a109d 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h @@ -296,6 +296,32 @@ class OpenXrInterface::Implementation // flag to check if the HTC VIVE Focus3 controllers are supported by the runtime. bool focus3_supported = false; + //flag to check if the HTC facial tracking is supported by the runtime. + bool htc_facial_tracking_extension_supported = false; + + //flag to check if facial tracking is supported by the headset. + bool htc_eye_facial_tracking_supported = false; + + //flag to check if lip facial tracking is supported by the headset. + bool htc_lip_facial_tracking_supported = false; + + // Pointer to function to create the HTC facial tracker + PFN_xrCreateFacialTrackerHTC pfn_xrCreateFacialTrackerHTC = NULL; + + // Pointer to function to destroy the HTC facial tracker + PFN_xrDestroyFacialTrackerHTC pfn_xrDestroyFacialTrackerHTC = NULL; + + // Pointer to function to get the facial expressions + PFN_xrGetFacialExpressionsHTC pfn_xrGetFacialExpressionsHTC = NULL; + + // Handles for eye and lip tracking + XrFacialTrackerHTC* htc_eye_facial_tracker = nullptr; + XrFacialTrackerHTC* htc_lip_facial_tracker = nullptr; + + // Stucts to store the facial expressions + std::vector htc_eye_expressions; + std::vector htc_lip_expressions; + // state of the application XrSessionState state = XR_SESSION_STATE_UNKNOWN; From f177e66cfeebf208cfccb190843ab017e4fdc9aa Mon Sep 17 00:00:00 2001 From: Stefano Date: Wed, 11 Sep 2024 16:27:57 +0200 Subject: [PATCH 04/13] Added streaming of expressions --- src/devices/openxrheadset/CMakeLists.txt | 2 + .../openxrheadset/ExpressionsManager.cpp | 64 +++++++++++++++++++ .../openxrheadset/ExpressionsManager.h | 36 +++++++++++ src/devices/openxrheadset/OpenXrHeadset.cpp | 7 ++ src/devices/openxrheadset/OpenXrHeadset.h | 3 + src/devices/openxrheadset/OpenXrInterface.cpp | 20 ++++++ src/devices/openxrheadset/OpenXrInterface.h | 8 +++ 7 files changed, 140 insertions(+) create mode 100644 src/devices/openxrheadset/ExpressionsManager.cpp create mode 100644 src/devices/openxrheadset/ExpressionsManager.h diff --git a/src/devices/openxrheadset/CMakeLists.txt b/src/devices/openxrheadset/CMakeLists.txt index a4b9d59..e7f32cc 100644 --- a/src/devices/openxrheadset/CMakeLists.txt +++ b/src/devices/openxrheadset/CMakeLists.txt @@ -80,6 +80,7 @@ set(yarp_openxrheadset_driver_SRCS PosePublisher.cpp CustomPosePublisher.cpp FilteredPosePublisher.cpp + ExpressionsManager.cpp ) set(yarp_openxrheadset_driver_HDRS @@ -91,6 +92,7 @@ set(yarp_openxrheadset_driver_HDRS PosePublisher.h CustomPosePublisher.h FilteredPosePublisher.h + ExpressionsManager.h ) set (THRIFTS thrifts/OpenXrHeadsetCommands.thrift) diff --git a/src/devices/openxrheadset/ExpressionsManager.cpp b/src/devices/openxrheadset/ExpressionsManager.cpp new file mode 100644 index 0000000..be90801 --- /dev/null +++ b/src/devices/openxrheadset/ExpressionsManager.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2024 Istituto Italiano di Tecnologia (IIT) + * All rights reserved. + * + * This software may be modified and distributed under the terms of the + * BSD-2-Clause license. See the accompanying LICENSE file for details. + */ + +#include + +bool ExpressionsManager::configure(const std::string& prefix, bool eyeSupported, bool lipSupported) +{ + m_eyeSupported = eyeSupported; + m_lipSupported = lipSupported; + + if (m_eyeSupported) + { + if (!m_eyeExpressionsPort.open(prefix + "/eyeExpressions")) + { + return false; + } + } + + if (m_lipSupported) + { + if (!m_lipExpressionsPort.open(prefix + "/lipExpressions")) + { + return false; + } + } + + return true; +} + +void ExpressionsManager::setExpressions(const std::vector& eyeExpressions, const std::vector& lipExpressions) +{ + if (m_eyeSupported) + { + yarp::sig::Vector& eyeExpressionsVector = m_eyeExpressionsPort.prepare(); + eyeExpressionsVector.resize(eyeExpressions.size()); + for (size_t i = 0; i < eyeExpressions.size(); ++i) + { + eyeExpressionsVector[i] = eyeExpressions[i]; + } + m_eyeExpressionsPort.write(); + } + + if (m_lipSupported) + { + yarp::sig::Vector& lipExpressionsVector = m_lipExpressionsPort.prepare(); + lipExpressionsVector.resize(lipExpressions.size()); + for (size_t i = 0; i < lipExpressions.size(); ++i) + { + lipExpressionsVector[i] = lipExpressions[i]; + } + m_lipExpressionsPort.write(); + } +} + +void ExpressionsManager::close() +{ + m_eyeExpressionsPort.close(); + m_lipExpressionsPort.close(); +} diff --git a/src/devices/openxrheadset/ExpressionsManager.h b/src/devices/openxrheadset/ExpressionsManager.h new file mode 100644 index 0000000..dc28dfc --- /dev/null +++ b/src/devices/openxrheadset/ExpressionsManager.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 Istituto Italiano di Tecnologia (IIT) + * All rights reserved. + * + * This software may be modified and distributed under the terms of the + * BSD-2-Clause license. See the accompanying LICENSE file for details. + */ + +#ifndef YARP_DEV_EXPRESSIONSMANAGER_H +#define YARP_DEV_EXPRESSIONSMANAGER_H + +#include +#include + +#include +#include + +class ExpressionsManager +{ + yarp::os::BufferedPort m_eyeExpressionsPort; + yarp::os::BufferedPort m_lipExpressionsPort; + bool m_eyeSupported{ false }; + bool m_lipSupported{ false }; + +public: + + bool configure(const std::string& prefix, bool eyeSupported, bool lipSupported); + + void setExpressions(const std::vector& eyeExpressions, const std::vector& lipExpressions); + + void close(); +}; + + + +#endif // YARP_DEV_EXPRESSIONSMANAGER_H \ No newline at end of file diff --git a/src/devices/openxrheadset/OpenXrHeadset.cpp b/src/devices/openxrheadset/OpenXrHeadset.cpp index c2798c8..e31db9a 100644 --- a/src/devices/openxrheadset/OpenXrHeadset.cpp +++ b/src/devices/openxrheadset/OpenXrHeadset.cpp @@ -462,6 +462,10 @@ bool yarp::dev::OpenXrHeadset::threadInit() slide.layer.setImage(slide.options.initialSlide); } + // We know if the expressions are supported only after the initialization of the OpenXrInterface + m_expressionsManager.configure(m_prefix, + m_openXrInterface.eyeExpressionsSupported(), + m_openXrInterface.lipExpressionsSupported()); this->yarp().attachAsServer(this->m_rpcPort); if (!m_rpcPort.open(m_rpcPortName)) { @@ -507,6 +511,7 @@ void yarp::dev::OpenXrHeadset::threadRelease() m_labels.clear(); m_slides.clear(); m_eyesManager.close(); + m_expressionsManager.close(); m_openXrInterface.close(); @@ -618,6 +623,8 @@ void yarp::dev::OpenXrHeadset::run() m_posesManager.setTransformFromRawToRootFrame(m_rootFrameRawHRootFrame); m_posesManager.publishFrames(); + + m_expressionsManager.setExpressions(m_openXrInterface.eyeExpressions(), m_openXrInterface.lipExpressions()); } else { diff --git a/src/devices/openxrheadset/OpenXrHeadset.h b/src/devices/openxrheadset/OpenXrHeadset.h index 4f23ebb..64e365f 100644 --- a/src/devices/openxrheadset/OpenXrHeadset.h +++ b/src/devices/openxrheadset/OpenXrHeadset.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -323,6 +324,8 @@ class yarp::dev::OpenXrHeadset : public yarp::dev::DeviceDriver, EyesManager m_eyesManager; + ExpressionsManager m_expressionsManager; + std::vector m_huds; std::vector m_labels; std::vector m_slides; diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index 73c5c8d..e1d2b25 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -1945,6 +1945,26 @@ bool OpenXrInterface::shouldResetLocalReferenceSpace() return shouldReset; } +bool OpenXrInterface::eyeExpressionsSupported() const +{ + return m_pimpl->htc_eye_facial_tracker; +} + +bool OpenXrInterface::lipExpressionsSupported() const +{ + return m_pimpl->htc_lip_facial_tracker; +} + +const std::vector& OpenXrInterface::eyeExpressions() const +{ + return m_pimpl->htc_eye_expressions; +} + +const std::vector& OpenXrInterface::lipExpressions() const +{ + return m_pimpl->htc_lip_expressions; +} + void OpenXrInterface::close() { m_pimpl->closing = true; diff --git a/src/devices/openxrheadset/OpenXrInterface.h b/src/devices/openxrheadset/OpenXrInterface.h index 66b32ed..2ed8470 100644 --- a/src/devices/openxrheadset/OpenXrInterface.h +++ b/src/devices/openxrheadset/OpenXrInterface.h @@ -197,6 +197,14 @@ class OpenXrInterface bool shouldResetLocalReferenceSpace(); + bool eyeExpressionsSupported() const; + + bool lipExpressionsSupported() const; + + const std::vector& eyeExpressions() const; + + const std::vector& lipExpressions() const; + void close(); }; From dd9b1f2bf407e59a6f3b036b2dcd628a9392de0a Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 13 Sep 2024 09:15:16 +0200 Subject: [PATCH 05/13] Added gaze extension --- src/devices/openxrheadset/OpenXrInterface.cpp | 74 ++++++++++++++++++- src/devices/openxrheadset/OpenXrInterface.h | 5 ++ .../openxrheadset/impl/OpenXrInterfaceImpl.h | 4 + 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index e1d2b25..2eafffa 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -40,6 +40,7 @@ bool OpenXrInterface::checkExtensions() bool opengl_supported = false; bool depth_supported = false; bool debug_supported = false; + bool gaze_supported = false; std::stringstream supported_extensions; supported_extensions << "Supported extensions: " <htc_facial_tracking_extension_supported = true; } + if (strcmp(XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME, ext_props[i].extensionName) == 0) { + gaze_supported = true; + } + supported_extensions << std::endl << " - " << ext_props[i].extensionName; } @@ -101,6 +106,11 @@ bool OpenXrInterface::checkExtensions() yCWarning(OPENXRHEADSET) << "Runtime does not support the HTC Vive Facial Tracking!"; } + if (!gaze_supported) { + yCWarning(OPENXRHEADSET) << "Runtime does not support the eye gaze extension!"; + m_pimpl->use_gaze = false; + } + return true; } @@ -353,6 +363,8 @@ void OpenXrInterface::checkSystemProperties() system_props.type = XR_TYPE_SYSTEM_PROPERTIES; system_props.next = NULL; + void** next_chain = &system_props.next; + XrSystemFacialTrackingPropertiesHTC facial_tracking_props; facial_tracking_props.type = XR_TYPE_SYSTEM_FACIAL_TRACKING_PROPERTIES_HTC; facial_tracking_props.next = NULL; @@ -361,7 +373,19 @@ void OpenXrInterface::checkSystemProperties() if (m_pimpl->htc_facial_tracking_extension_supported) { - system_props.next = &facial_tracking_props; + *next_chain = &facial_tracking_props; + next_chain = &facial_tracking_props.next; + } + + XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_props; + eye_gaze_props.type = XR_TYPE_SYSTEM_EYE_GAZE_INTERACTION_PROPERTIES_EXT; + eye_gaze_props.next = NULL; + eye_gaze_props.supportsEyeGazeInteraction = XR_FALSE; + + if (m_pimpl->use_gaze) + { + *next_chain = &eye_gaze_props; + next_chain = &eye_gaze_props.next; } XrResult result = xrGetSystemProperties(m_pimpl->instance, m_pimpl->system_id, &system_props); @@ -388,6 +412,14 @@ void OpenXrInterface::checkSystemProperties() m_pimpl->htc_eye_facial_tracking_supported = facial_tracking_props.supportEyeFacialTracking; m_pimpl->htc_lip_facial_tracking_supported = facial_tracking_props.supportLipFacialTracking; } + + if (m_pimpl->use_gaze) + { + yCInfo(OPENXRHEADSET, "Eye gaze properties for system %lu: Eye gaze %d", + system_props.systemId, eye_gaze_props.supportsEyeGazeInteraction); + m_pimpl->use_gaze = eye_gaze_props.supportsEyeGazeInteraction; + } + } bool OpenXrInterface::prepareGL() @@ -879,6 +911,24 @@ bool OpenXrInterface::prepareXrActions() interactionProfilesPrefixes.push_back({HTC_VIVE_TRACKER_INTERACTION_PROFILE_TAG, "vive_tracker_"}); } + if (m_pimpl->use_gaze) + { + InputActionsDeclaration gazeInputs; + + gazeInputs.poses = + { + {"/input/gaze_ext/pose", "pose"} + }; + + TopLevelPathDeclaration gaze; + gaze.stringPath = "/user/eyes_ext"; + gaze.actionNamePrefix = "eyes_"; + gaze.inputsDeclarations[GAZE_INTERACTION_PROFILE_TAG] = gazeInputs; + + topLevelPathsDeclaration.push_back(gaze); //The gaze should be the last one in this list + interactionProfilesPrefixes.push_back({ GAZE_INTERACTION_PROFILE_TAG, "gaze_" }); + } + if (!m_pimpl->fillActionBindings(interactionProfilesPrefixes, topLevelPathsDeclaration)) return false; @@ -1575,6 +1625,7 @@ bool OpenXrInterface::initialize(const OpenXrInterfaceSettings &settings) m_pimpl->hideWindow = settings.hideWindow; m_pimpl->renderInPlaySpace = settings.renderInPlaySpace; + m_pimpl->use_gaze = settings.useGaze; #ifdef DEBUG_RENDERING_LOCATION m_pimpl->renderInPlaySpace = true; @@ -1965,6 +2016,27 @@ const std::vector& OpenXrInterface::lipExpressions() const return m_pimpl->htc_lip_expressions; } +bool OpenXrInterface::gazeSupported() const +{ + return m_pimpl->use_gaze; +} + +OpenXrInterface::Pose OpenXrInterface::gazePose() const +{ + if (!m_pimpl->use_gaze) + { + yCError(OPENXRHEADSET) << "Gaze is not enabled."; + return Pose(); + } + const std::vector& currentPoses = m_pimpl->top_level_paths.back().currentActions().poses; //eyes are in the last position + if (currentPoses.size() == 0) //no pose in the current interaction profile + { + return OpenXrInterface::Pose(); + } + + return currentPoses.front().pose; +} + void OpenXrInterface::close() { m_pimpl->closing = true; diff --git a/src/devices/openxrheadset/OpenXrInterface.h b/src/devices/openxrheadset/OpenXrInterface.h index 2ed8470..fcb572d 100644 --- a/src/devices/openxrheadset/OpenXrInterface.h +++ b/src/devices/openxrheadset/OpenXrInterface.h @@ -66,6 +66,7 @@ struct OpenXrInterfaceSettings double posesPredictionInMs{0.0}; bool hideWindow{false}; bool renderInPlaySpace{false}; + bool useGaze{ true }; }; class OpenXrInterface @@ -205,6 +206,10 @@ class OpenXrInterface const std::vector& lipExpressions() const; + bool gazeSupported() const; + + Pose gazePose() const; + void close(); }; diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h index a4a109d..4db354f 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h @@ -35,6 +35,7 @@ #define HTC_VIVE_INTERACTION_PROFILE_TAG "/interaction_profiles/htc/vive_controller" #define HTC_VIVE_TRACKER_INTERACTION_PROFILE_TAG "/interaction_profiles/htc/vive_tracker_htcx" #define HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_PROFILE_TAG "/interaction_profiles/htc/vive_focus3_controller" +#define GAZE_INTERACTION_PROFILE_TAG "/interaction_profiles/ext/eye_gaze_interaction" // STRUCTS template @@ -322,6 +323,9 @@ class OpenXrInterface::Implementation std::vector htc_eye_expressions; std::vector htc_lip_expressions; + // Flag to enable the use of gaze + bool use_gaze = true; + // state of the application XrSessionState state = XR_SESSION_STATE_UNKNOWN; From 57a37d02f604ea05588be66160e6135d385214eb Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 15 Nov 2024 12:35:01 +0100 Subject: [PATCH 06/13] Added streaming of gaze position in head frame --- .../openxrheadset/ExpressionsManager.cpp | 33 +++++++++++++++++-- .../openxrheadset/ExpressionsManager.h | 7 +++- src/devices/openxrheadset/OpenXrHeadset.cpp | 7 +++- src/devices/openxrheadset/OpenXrInterface.cpp | 1 - 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/devices/openxrheadset/ExpressionsManager.cpp b/src/devices/openxrheadset/ExpressionsManager.cpp index be90801..10968c0 100644 --- a/src/devices/openxrheadset/ExpressionsManager.cpp +++ b/src/devices/openxrheadset/ExpressionsManager.cpp @@ -8,14 +8,15 @@ #include -bool ExpressionsManager::configure(const std::string& prefix, bool eyeSupported, bool lipSupported) +bool ExpressionsManager::configure(const std::string& prefix, bool eyeSupported, bool lipSupported, bool gazeSupported) { m_eyeSupported = eyeSupported; m_lipSupported = lipSupported; + m_gazeSupported = gazeSupported; if (m_eyeSupported) { - if (!m_eyeExpressionsPort.open(prefix + "/eyeExpressions")) + if (!m_eyeExpressionsPort.open(prefix + "/expressions/eye")) { return false; } @@ -23,7 +24,15 @@ bool ExpressionsManager::configure(const std::string& prefix, bool eyeSupported, if (m_lipSupported) { - if (!m_lipExpressionsPort.open(prefix + "/lipExpressions")) + if (!m_lipExpressionsPort.open(prefix + "/expressions/lip")) + { + return false; + } + } + + if (m_gazeSupported) + { + if (!m_gazePort.open(prefix + "/expressions/gaze")) { return false; } @@ -57,8 +66,26 @@ void ExpressionsManager::setExpressions(const std::vector& eyeExpressions } } +void ExpressionsManager::setGaze(const OpenXrInterface::Pose& headPose, const OpenXrInterface::Pose& gaze) +{ + if (!m_gazeSupported || !gaze.positionValid || !headPose.positionValid || !headPose.rotationValid) + { + return; + } + + Eigen::Vector3f gazeInHead = headPose.rotation.inverse() * (gaze.position - headPose.position); + + yarp::sig::Vector& gazeVector = m_gazePort.prepare(); + gazeVector.resize(3); + gazeVector[0] = gazeInHead.x(); + gazeVector[1] = gazeInHead.y(); + gazeVector[2] = gazeInHead.z(); + m_gazePort.write(); +} + void ExpressionsManager::close() { m_eyeExpressionsPort.close(); m_lipExpressionsPort.close(); + m_gazePort.close(); } diff --git a/src/devices/openxrheadset/ExpressionsManager.h b/src/devices/openxrheadset/ExpressionsManager.h index dc28dfc..470c1bd 100644 --- a/src/devices/openxrheadset/ExpressionsManager.h +++ b/src/devices/openxrheadset/ExpressionsManager.h @@ -14,20 +14,25 @@ #include #include +#include class ExpressionsManager { yarp::os::BufferedPort m_eyeExpressionsPort; yarp::os::BufferedPort m_lipExpressionsPort; + yarp::os::BufferedPort m_gazePort; bool m_eyeSupported{ false }; bool m_lipSupported{ false }; + bool m_gazeSupported{ false }; public: - bool configure(const std::string& prefix, bool eyeSupported, bool lipSupported); + bool configure(const std::string& prefix, bool eyeSupported, bool lipSupported, bool gazeSupported); void setExpressions(const std::vector& eyeExpressions, const std::vector& lipExpressions); + void setGaze(const OpenXrInterface::Pose& headPose, const OpenXrInterface::Pose& gaze); + void close(); }; diff --git a/src/devices/openxrheadset/OpenXrHeadset.cpp b/src/devices/openxrheadset/OpenXrHeadset.cpp index e31db9a..eb40c98 100644 --- a/src/devices/openxrheadset/OpenXrHeadset.cpp +++ b/src/devices/openxrheadset/OpenXrHeadset.cpp @@ -262,6 +262,8 @@ bool yarp::dev::OpenXrHeadset::open(yarp::os::Searchable &cfg) m_openXrInterfaceSettings.posesPredictionInMs = cfg.check("vr_poses_prediction_in_ms", yarp::os::Value(0.0)).asFloat64(); m_openXrInterfaceSettings.hideWindow = (m_useNativeQuadLayers && !cfg.check("hide_window")) || (cfg.check("hide_window") && (cfg.find("hide_window").isNull() || cfg.find("hide_window").asBool())); m_openXrInterfaceSettings.renderInPlaySpace = cfg.check("render_in_play_space") && (cfg.find("render_in_play_space").isNull() || cfg.find("render_in_play_space").asBool()); + bool noGaze = cfg.check("no_gaze") && (cfg.find("no_gaze").isNull() || cfg.find("no_gaze").asBool()); + m_openXrInterfaceSettings.useGaze = !noGaze; m_getStickAsAxis = cfg.check("stick_as_axis", yarp::os::Value(false)).asBool(); m_rootFrame = cfg.check("tf_root_frame", yarp::os::Value("openxr_origin")).asString(); @@ -465,7 +467,9 @@ bool yarp::dev::OpenXrHeadset::threadInit() // We know if the expressions are supported only after the initialization of the OpenXrInterface m_expressionsManager.configure(m_prefix, m_openXrInterface.eyeExpressionsSupported(), - m_openXrInterface.lipExpressionsSupported()); + m_openXrInterface.lipExpressionsSupported(), + m_openXrInterface.gazeSupported()); + this->yarp().attachAsServer(this->m_rpcPort); if (!m_rpcPort.open(m_rpcPortName)) { @@ -625,6 +629,7 @@ void yarp::dev::OpenXrHeadset::run() m_posesManager.publishFrames(); m_expressionsManager.setExpressions(m_openXrInterface.eyeExpressions(), m_openXrInterface.lipExpressions()); + m_expressionsManager.setGaze(m_openXrInterface.headPose(), m_openXrInterface.gazePose()); } else { diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index 2eafffa..cd0329f 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -2025,7 +2025,6 @@ OpenXrInterface::Pose OpenXrInterface::gazePose() const { if (!m_pimpl->use_gaze) { - yCError(OPENXRHEADSET) << "Gaze is not enabled."; return Pose(); } const std::vector& currentPoses = m_pimpl->top_level_paths.back().currentActions().poses; //eyes are in the last position From 1deaed59be3de695f6e2f1e092c7ba77d0350cb4 Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 13 Sep 2024 10:39:03 +0200 Subject: [PATCH 07/13] Added commands to get the expressions ports and the IPD --- .../openxrheadset/ExpressionsManager.cpp | 24 +++++++-- .../openxrheadset/ExpressionsManager.h | 9 ++++ src/devices/openxrheadset/OpenXrHeadset.cpp | 43 ++++++++++++++++ src/devices/openxrheadset/OpenXrHeadset.h | 44 ++++++++++++++++- src/devices/openxrheadset/OpenXrInterface.cpp | 7 +++ src/devices/openxrheadset/OpenXrInterface.h | 2 + .../openxrheadset/impl/OpenXrInterfaceImpl.h | 2 + .../thrifts/OpenXrHeadsetCommands.thrift | 49 +++++++++++++++++-- 8 files changed, 173 insertions(+), 7 deletions(-) diff --git a/src/devices/openxrheadset/ExpressionsManager.cpp b/src/devices/openxrheadset/ExpressionsManager.cpp index 10968c0..09edecf 100644 --- a/src/devices/openxrheadset/ExpressionsManager.cpp +++ b/src/devices/openxrheadset/ExpressionsManager.cpp @@ -13,10 +13,13 @@ bool ExpressionsManager::configure(const std::string& prefix, bool eyeSupported, m_eyeSupported = eyeSupported; m_lipSupported = lipSupported; m_gazeSupported = gazeSupported; + m_eyeExpressionsPortName = prefix + "/expressions/eye"; + m_lipExpressionsPortName = prefix + "/expressions/lip"; + m_gazePortName = prefix + "/expressions/gaze"; if (m_eyeSupported) { - if (!m_eyeExpressionsPort.open(prefix + "/expressions/eye")) + if (!m_eyeExpressionsPort.open(m_eyeExpressionsPortName)) { return false; } @@ -24,7 +27,7 @@ bool ExpressionsManager::configure(const std::string& prefix, bool eyeSupported, if (m_lipSupported) { - if (!m_lipExpressionsPort.open(prefix + "/expressions/lip")) + if (!m_lipExpressionsPort.open(m_lipExpressionsPortName)) { return false; } @@ -32,7 +35,7 @@ bool ExpressionsManager::configure(const std::string& prefix, bool eyeSupported, if (m_gazeSupported) { - if (!m_gazePort.open(prefix + "/expressions/gaze")) + if (!m_gazePort.open(m_gazePortName)) { return false; } @@ -89,3 +92,18 @@ void ExpressionsManager::close() m_lipExpressionsPort.close(); m_gazePort.close(); } + +std::string ExpressionsManager::getEyeExpressionsPortName() const +{ + return m_eyeExpressionsPortName; +} + +std::string ExpressionsManager::getLipExpressionsPortName() const +{ + return m_lipExpressionsPortName; +} + +std::string ExpressionsManager::getGazePortName() const +{ + return m_gazePortName; +} diff --git a/src/devices/openxrheadset/ExpressionsManager.h b/src/devices/openxrheadset/ExpressionsManager.h index 470c1bd..5c71929 100644 --- a/src/devices/openxrheadset/ExpressionsManager.h +++ b/src/devices/openxrheadset/ExpressionsManager.h @@ -24,6 +24,9 @@ class ExpressionsManager bool m_eyeSupported{ false }; bool m_lipSupported{ false }; bool m_gazeSupported{ false }; + std::string m_eyeExpressionsPortName; + std::string m_lipExpressionsPortName; + std::string m_gazePortName; public: @@ -34,6 +37,12 @@ class ExpressionsManager void setGaze(const OpenXrInterface::Pose& headPose, const OpenXrInterface::Pose& gaze); void close(); + + std::string getEyeExpressionsPortName() const; + + std::string getLipExpressionsPortName() const; + + std::string getGazePortName() const; }; diff --git a/src/devices/openxrheadset/OpenXrHeadset.cpp b/src/devices/openxrheadset/OpenXrHeadset.cpp index eb40c98..6974573 100644 --- a/src/devices/openxrheadset/OpenXrHeadset.cpp +++ b/src/devices/openxrheadset/OpenXrHeadset.cpp @@ -975,6 +975,13 @@ bool yarp::dev::OpenXrHeadset::setInterCameraDistance(const double distance) return m_eyesManager.setInterCameraDistance(distance); } +double yarp::dev::OpenXrHeadset::getIPD() +{ + std::lock_guard lock(m_mutex); + + return m_openXrInterface.ipd(); +} + std::string yarp::dev::OpenXrHeadset::getLeftImageControlPortName() { std::lock_guard lock(m_mutex); @@ -1072,6 +1079,42 @@ bool yarp::dev::OpenXrHeadset::restartJoypadControlServer() return startJoypadControlServer(); } +bool yarp::dev::OpenXrHeadset::eyeExpressionsEnabled() +{ + std::lock_guard lock(m_mutex); + return m_openXrInterface.eyeExpressionsSupported(); +} + +std::string yarp::dev::OpenXrHeadset::getEyeExpressionsPortName() +{ + std::lock_guard lock(m_mutex); + return m_expressionsManager.getEyeExpressionsPortName(); +} + +bool yarp::dev::OpenXrHeadset::lipExpressionsEnabled() +{ + std::lock_guard lock(m_mutex); + return m_openXrInterface.lipExpressionsSupported(); +} + +std::string yarp::dev::OpenXrHeadset::getLipExpressionsPortName() +{ + std::lock_guard lock(m_mutex); + return m_expressionsManager.getLipExpressionsPortName(); +} + +bool yarp::dev::OpenXrHeadset::gazeEnabled() +{ + std::lock_guard lock(m_mutex); + return m_openXrInterface.gazeSupported(); +} + +std::string yarp::dev::OpenXrHeadset::getGazePortName() +{ + std::lock_guard lock(m_mutex); + return m_expressionsManager.getGazePortName(); +} + bool yarp::dev::OpenXrHeadset::startJoypadControlServer() { stopJoypadControlServer(); diff --git a/src/devices/openxrheadset/OpenXrHeadset.h b/src/devices/openxrheadset/OpenXrHeadset.h index 64e365f..557d62b 100644 --- a/src/devices/openxrheadset/OpenXrHeadset.h +++ b/src/devices/openxrheadset/OpenXrHeadset.h @@ -191,7 +191,7 @@ class yarp::dev::OpenXrHeadset : public yarp::dev::DeviceDriver, /** * Get the current lateral distance between the visualization of the robot cameras. - * @return The IPD in meters. + * @return The distance in meters. */ virtual double getInterCameraDistance() override; @@ -202,6 +202,12 @@ class yarp::dev::OpenXrHeadset : public yarp::dev::DeviceDriver, */ virtual bool setInterCameraDistance(const double distance) override; + /** + * Get the current IPD (Inter Pupillary Distance) of the VR eyes. + * @return The IPD in meters + */ + virtual double getIPD() override; + /** * Get the name of the port trough which it is possible to control the left image. * @return the name of the port to control the left image. @@ -271,6 +277,42 @@ class yarp::dev::OpenXrHeadset : public yarp::dev::DeviceDriver, */ virtual bool restartJoypadControlServer() override; + /** + * Check if the eye expressions are enabled + * @return True if the eye expressions are enabled, false otherwise + */ + virtual bool eyeExpressionsEnabled() override; + + /** + * Get the name of the port trough which it is possible to get the eye expressions. + * @return the name of the port to get the eye expressions. + */ + virtual std::string getEyeExpressionsPortName() override; + + /** + * Check if the lip expressions are enabled + * @return True if the lip expressions are enabled, false otherwise + */ + virtual bool lipExpressionsEnabled() override; + + /** + * Get the name of the port trough which it is possible to get the lip expressions. + * @return the name of the port to get the lip expressions. + */ + virtual std::string getLipExpressionsPortName() override; + + /** + * Check if the gaze acquisition is enabled + * @return True if the gaze acquisition is enabled, false otherwise + */ + virtual bool gazeEnabled() override; + + /** + * Get the name of the port trough which it is possible to get the gaze position. + * @return the name of the port to get the gaze position. + */ + virtual std::string getGazePortName() override; + private: /** diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index cd0329f..64e76f1 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -1127,6 +1127,8 @@ void OpenXrInterface::updateXrSpaces() m_pimpl->mid_views_pose_inverted.orientation = toXr(toEigen(m_pimpl->mid_views_pose.orientation).inverse()); m_pimpl->mid_views_pose_inverted.position = toXr(toEigen(m_pimpl->mid_views_pose_inverted.orientation) * -toEigen(m_pimpl->mid_views_pose.position)); + m_pimpl->ipd = (toEigen(m_pimpl->views[1].pose.position) - toEigen(m_pimpl->views[0].pose.position)).norm(); + for (size_t i = 0; i < m_pimpl->views.size(); ++i) { #ifdef DEBUG_RENDERING_LOCATION @@ -1794,6 +1796,11 @@ bool OpenXrInterface::isRunning() const return m_pimpl->initialized && !m_pimpl->closing; } +float OpenXrInterface::ipd() const +{ + return m_pimpl->ipd; +} + OpenXrInterface::Pose OpenXrInterface::headPose() const { return XrSpaceLocationToPose(m_pimpl->view_space_location); diff --git a/src/devices/openxrheadset/OpenXrInterface.h b/src/devices/openxrheadset/OpenXrInterface.h index fcb572d..5708e28 100644 --- a/src/devices/openxrheadset/OpenXrInterface.h +++ b/src/devices/openxrheadset/OpenXrInterface.h @@ -170,6 +170,8 @@ class OpenXrInterface bool isRunning() const; + float ipd() const; + Pose headPose() const; Velocity headVelocity() const; diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h index 4db354f..cc88b95 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h @@ -279,6 +279,8 @@ class OpenXrInterface::Implementation // position of a frame in the middle of the eyes, oriented as the first eye XrPosef mid_views_pose_inverted; + float ipd = 0.06f; + // List of top level paths to retrieve the state of each action std::vector top_level_paths; diff --git a/src/devices/openxrheadset/thrifts/OpenXrHeadsetCommands.thrift b/src/devices/openxrheadset/thrifts/OpenXrHeadsetCommands.thrift index 49d256f..a772b8f 100644 --- a/src/devices/openxrheadset/thrifts/OpenXrHeadsetCommands.thrift +++ b/src/devices/openxrheadset/thrifts/OpenXrHeadsetCommands.thrift @@ -114,7 +114,7 @@ service OpenXrHeadsetCommands /** * Get the current lateral distance between the visualization of the robot cameras. - * @return The IPD in meters. + * @return The distance in meters. */ double getInterCameraDistance(); @@ -125,6 +125,12 @@ service OpenXrHeadsetCommands */ bool setInterCameraDistance(1:double distance); + /** + * Get the current IPD (Inter Pupillary Distance) of the VR eyes. + * @return The IPD in meters + */ + double getIPD(); + /** * Get the name of the port trough which it is possible to control the left image. * @return the name of the port to control the left image. @@ -180,7 +186,7 @@ service OpenXrHeadsetCommands */ bool setCustomPoseRelativeOrientation(1:string customFrameName, 2:double angle1, 3:double angle2, 4:double angle3); - /** + /** * Reset the transforms all the published tranforms. * This will also delete all the transforms currently stored in the transform server, * so also the static poses will be published again. This must be used with caution, @@ -188,9 +194,46 @@ service OpenXrHeadsetCommands */ bool resetTransforms(); - /** + /** * Start the joypad control server. The server will restart if already started. * @return True if the server is started successfully, false otherwise. */ + bool restartJoypadControlServer(); + + /** + * Check if the eye expressions are enabled + * @return True if the eye expressions are enabled, false otherwise + */ + bool eyeExpressionsEnabled(); + + /** + * Get the name of the port trough which it is possible to get the eye expressions. + * @return the name of the port to get the eye expressions. + */ + string getEyeExpressionsPortName(); + + /** + * Check if the lip expressions are enabled + * @return True if the lip expressions are enabled, false otherwise + */ + bool lipExpressionsEnabled(); + + /** + * Get the name of the port trough which it is possible to get the lip expressions. + * @return the name of the port to get the lip expressions. + */ + string getLipExpressionsPortName(); + + /** + * Check if the gaze acquisition is enabled + * @return True if the gaze acquisition is enabled, false otherwise + */ + bool gazeEnabled(); + + /** + * Get the name of the port trough which it is possible to get the gaze position. + * @return the name of the port to get the gaze position. + */ + string getGazePortName(); } From 00719c41382b0f7128640a892bb7031bb2a8a0ab Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 13 Sep 2024 18:33:23 +0200 Subject: [PATCH 08/13] Added gaze extension in the required extensions and avoid to disable it if the property is not set --- src/devices/openxrheadset/OpenXrInterface.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index 64e76f1..28d88e1 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -158,6 +158,10 @@ bool OpenXrInterface::prepareXrInstance() { requestedExtensions.push_back(XR_HTC_FACIAL_TRACKING_EXTENSION_NAME); } + if (m_pimpl->use_gaze) + { + requestedExtensions.push_back(XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME); + } // Populate the info to create the instance XrInstanceCreateInfo instanceCreateInfo @@ -417,7 +421,10 @@ void OpenXrInterface::checkSystemProperties() { yCInfo(OPENXRHEADSET, "Eye gaze properties for system %lu: Eye gaze %d", system_props.systemId, eye_gaze_props.supportsEyeGazeInteraction); - m_pimpl->use_gaze = eye_gaze_props.supportsEyeGazeInteraction; + if (!eye_gaze_props.supportsEyeGazeInteraction) + { + yCWarning(OPENXRHEADSET) << "The runtime does not seem to support eye gaze interaction! Trying to use it anyway."; + } } } From 4aa8be304890d958ccb3784900c50afe65045699 Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 13 Sep 2024 18:43:11 +0200 Subject: [PATCH 09/13] Disable jump filter for eye gaze --- src/devices/openxrheadset/FilteredPosePublisher.cpp | 11 +++++++++-- src/devices/openxrheadset/OpenXrInterface.cpp | 2 +- src/devices/openxrheadset/OpenXrInterface.h | 7 +++++++ .../openxrheadset/impl/OpenXrInterfaceImpl.cpp | 1 + src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h | 7 ++++++- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/devices/openxrheadset/FilteredPosePublisher.cpp b/src/devices/openxrheadset/FilteredPosePublisher.cpp index 7cb9d95..116021f 100644 --- a/src/devices/openxrheadset/FilteredPosePublisher.cpp +++ b/src/devices/openxrheadset/FilteredPosePublisher.cpp @@ -146,6 +146,13 @@ void FilteredPosePublisher::updateInputPose(const OpenXrInterface::NamedPoseVelo { return; } - - PosePublisher::updateInputPose(filterJumps(input)); + switch (input.filterType) + { + case PoseFilterType::JumpFilter: + PosePublisher::updateInputPose(filterJumps(input)); + break; + default: + PosePublisher::updateInputPose(input); + break; + } } diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index 28d88e1..20c8ab1 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -924,7 +924,7 @@ bool OpenXrInterface::prepareXrActions() gazeInputs.poses = { - {"/input/gaze_ext/pose", "pose"} + {"/input/gaze_ext/pose", "pose", PoseFilterType::None} }; TopLevelPathDeclaration gaze; diff --git a/src/devices/openxrheadset/OpenXrInterface.h b/src/devices/openxrheadset/OpenXrInterface.h index 5708e28..a7a615b 100644 --- a/src/devices/openxrheadset/OpenXrInterface.h +++ b/src/devices/openxrheadset/OpenXrInterface.h @@ -61,6 +61,12 @@ class IOpenXrQuadLayer virtual void setEnabled(bool enabled) = 0; }; +enum class PoseFilterType +{ + None, + JumpFilter, +}; + struct OpenXrInterfaceSettings { double posesPredictionInMs{0.0}; @@ -142,6 +148,7 @@ class OpenXrInterface std::string name; Pose pose; Velocity velocity; + PoseFilterType filterType; static NamedPoseVelocity Identity(const std::string& name); }; diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.cpp b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.cpp index 08b60b6..7e62ff6 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.cpp +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.cpp @@ -289,6 +289,7 @@ bool OpenXrInterface::Implementation::fillActionBindings(const std::vector poses; + std::vector poses; std::vector buttons; From f84019d3f364b6921b01f8e2007139938cdc8c43 Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 16 Sep 2024 13:17:09 +0200 Subject: [PATCH 10/13] Using facial tracking handles in a different way --- src/devices/openxrheadset/OpenXrInterface.cpp | 32 +++++++++---------- .../openxrheadset/impl/OpenXrInterfaceImpl.h | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index 20c8ab1..cb500f3 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -530,10 +530,10 @@ bool OpenXrInterface::prepareXrSession() .next = NULL, .facialTrackingType = XR_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC, }; - result = m_pimpl->pfn_xrCreateFacialTrackerHTC(m_pimpl->session, &facial_tracker_create_info, m_pimpl->htc_eye_facial_tracker); + result = m_pimpl->pfn_xrCreateFacialTrackerHTC(m_pimpl->session, &facial_tracker_create_info, &m_pimpl->htc_eye_facial_tracker); if (!m_pimpl->checkXrOutput(result, "Failed to create eye facial tracker! Avoiding using it.")) { - m_pimpl->htc_eye_facial_tracker = nullptr; + m_pimpl->htc_eye_facial_tracking_supported = false; } else { @@ -549,10 +549,10 @@ bool OpenXrInterface::prepareXrSession() .next = NULL, .facialTrackingType = XR_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC, }; - result = m_pimpl->pfn_xrCreateFacialTrackerHTC(m_pimpl->session, &facial_tracker_create_info, m_pimpl->htc_lip_facial_tracker); + result = m_pimpl->pfn_xrCreateFacialTrackerHTC(m_pimpl->session, &facial_tracker_create_info, &m_pimpl->htc_lip_facial_tracker); if (!m_pimpl->checkXrOutput(result, "Failed to create lip facial tracker! Avoiding using it.")) { - m_pimpl->htc_lip_facial_tracker = nullptr; + m_pimpl->htc_lip_facial_tracking_supported = false; } else { @@ -1203,7 +1203,7 @@ void OpenXrInterface::updateXrActions() } } - if (m_pimpl->htc_eye_facial_tracker) + if (m_pimpl->htc_eye_facial_tracking_supported) { XrFacialExpressionsHTC expressions = { .type = XR_TYPE_FACIAL_EXPRESSIONS_HTC, .next = NULL, @@ -1211,7 +1211,7 @@ void OpenXrInterface::updateXrActions() .sampleTime = m_pimpl->frame_state.predictedDisplayTime, .expressionCount = static_cast(m_pimpl->htc_eye_expressions.size()), .expressionWeightings = m_pimpl->htc_eye_expressions.data() }; - result = m_pimpl->pfn_xrGetFacialExpressionsHTC(*m_pimpl->htc_eye_facial_tracker, &expressions); + result = m_pimpl->pfn_xrGetFacialExpressionsHTC(m_pimpl->htc_eye_facial_tracker, &expressions); if (!m_pimpl->checkXrOutput(result, "Failed to get the facial expressions of the eye tracker!") || !expressions.isActive) { if (!expressions.isActive) @@ -1222,7 +1222,7 @@ void OpenXrInterface::updateXrActions() } } - if (m_pimpl->htc_lip_facial_tracker) + if (m_pimpl->htc_lip_facial_tracking_supported) { XrFacialExpressionsHTC expressions = { .type = XR_TYPE_FACIAL_EXPRESSIONS_HTC, .next = NULL, @@ -1230,7 +1230,7 @@ void OpenXrInterface::updateXrActions() .sampleTime = m_pimpl->frame_state.predictedDisplayTime, .expressionCount = static_cast(m_pimpl->htc_lip_expressions.size()), .expressionWeightings = m_pimpl->htc_lip_expressions.data() }; - result = m_pimpl->pfn_xrGetFacialExpressionsHTC(*m_pimpl->htc_lip_facial_tracker, &expressions); + result = m_pimpl->pfn_xrGetFacialExpressionsHTC(m_pimpl->htc_lip_facial_tracker, &expressions); if (!m_pimpl->checkXrOutput(result, "Failed to get the facial expressions of the lip tracker!") || !expressions.isActive) { if (!expressions.isActive) @@ -2012,12 +2012,12 @@ bool OpenXrInterface::shouldResetLocalReferenceSpace() bool OpenXrInterface::eyeExpressionsSupported() const { - return m_pimpl->htc_eye_facial_tracker; + return m_pimpl->htc_eye_facial_tracking_supported; } bool OpenXrInterface::lipExpressionsSupported() const { - return m_pimpl->htc_lip_facial_tracker; + return m_pimpl->htc_lip_facial_tracking_supported; } const std::vector& OpenXrInterface::eyeExpressions() const @@ -2054,16 +2054,16 @@ void OpenXrInterface::close() { m_pimpl->closing = true; - if (m_pimpl->htc_eye_facial_tracker) + if (m_pimpl->htc_eye_facial_tracking_supported) { - m_pimpl->pfn_xrDestroyFacialTrackerHTC(*m_pimpl->htc_eye_facial_tracker); - m_pimpl->htc_eye_facial_tracker = nullptr; + m_pimpl->pfn_xrDestroyFacialTrackerHTC(m_pimpl->htc_eye_facial_tracker); + m_pimpl->htc_eye_facial_tracking_supported = false; } - if (m_pimpl->htc_lip_facial_tracker) + if (m_pimpl->htc_lip_facial_tracking_supported) { - m_pimpl->pfn_xrDestroyFacialTrackerHTC(*m_pimpl->htc_lip_facial_tracker); - m_pimpl->htc_lip_facial_tracker = nullptr; + m_pimpl->pfn_xrDestroyFacialTrackerHTC(m_pimpl->htc_lip_facial_tracker); + m_pimpl->htc_lip_facial_tracking_supported = false; } if (m_pimpl->glFrameBufferId != 0) { diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h index bcf95db..412f8b1 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h @@ -323,8 +323,8 @@ class OpenXrInterface::Implementation PFN_xrGetFacialExpressionsHTC pfn_xrGetFacialExpressionsHTC = NULL; // Handles for eye and lip tracking - XrFacialTrackerHTC* htc_eye_facial_tracker = nullptr; - XrFacialTrackerHTC* htc_lip_facial_tracker = nullptr; + XrFacialTrackerHTC htc_eye_facial_tracker = XR_NULL_HANDLE; + XrFacialTrackerHTC htc_lip_facial_tracker = XR_NULL_HANDLE; // Stucts to store the facial expressions std::vector htc_eye_expressions; From 309990840b0481d7c4d6d40f8360969deb9a7f50 Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 16 Sep 2024 14:43:51 +0200 Subject: [PATCH 11/13] Added flag to disable use of expressions --- src/devices/openxrheadset/OpenXrHeadset.cpp | 3 +++ src/devices/openxrheadset/OpenXrInterface.cpp | 15 +++++++++------ src/devices/openxrheadset/OpenXrInterface.h | 1 + .../openxrheadset/impl/OpenXrInterfaceImpl.h | 6 +++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/devices/openxrheadset/OpenXrHeadset.cpp b/src/devices/openxrheadset/OpenXrHeadset.cpp index 6974573..451db79 100644 --- a/src/devices/openxrheadset/OpenXrHeadset.cpp +++ b/src/devices/openxrheadset/OpenXrHeadset.cpp @@ -265,6 +265,9 @@ bool yarp::dev::OpenXrHeadset::open(yarp::os::Searchable &cfg) bool noGaze = cfg.check("no_gaze") && (cfg.find("no_gaze").isNull() || cfg.find("no_gaze").asBool()); m_openXrInterfaceSettings.useGaze = !noGaze; + bool noExpressions = cfg.check("no_expressions") && (cfg.find("no_expressions").isNull() || cfg.find("no_expressions").asBool()); + m_openXrInterfaceSettings.useExpressions = !noExpressions; + m_getStickAsAxis = cfg.check("stick_as_axis", yarp::os::Value(false)).asBool(); m_rootFrame = cfg.check("tf_root_frame", yarp::os::Value("openxr_origin")).asString(); m_rootFrameRaw = m_rootFrame + "_raw"; diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index cb500f3..a209a06 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -41,6 +41,7 @@ bool OpenXrInterface::checkExtensions() bool depth_supported = false; bool debug_supported = false; bool gaze_supported = false; + bool htc_facial_tracking_supported = false; std::stringstream supported_extensions; supported_extensions << "Supported extensions: " <htc_facial_tracking_extension_supported = true; + htc_facial_tracking_supported = true; } if (strcmp(XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME, ext_props[i].extensionName) == 0) { @@ -102,8 +103,9 @@ bool OpenXrInterface::checkExtensions() yCWarning(OPENXRHEADSET) << "Runtime does not support the HTC Vive Focus 3 controllers!"; } - if (!m_pimpl->htc_facial_tracking_extension_supported) { + if (!htc_facial_tracking_supported) { yCWarning(OPENXRHEADSET) << "Runtime does not support the HTC Vive Facial Tracking!"; + m_pimpl->use_expressions = false; } if (!gaze_supported) { @@ -154,7 +156,7 @@ bool OpenXrInterface::prepareXrInstance() { requestedExtensions.push_back(XR_HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_EXTENSION_NAME); } - if (m_pimpl->htc_facial_tracking_extension_supported) + if (m_pimpl->use_expressions) { requestedExtensions.push_back(XR_HTC_FACIAL_TRACKING_EXTENSION_NAME); } @@ -255,7 +257,7 @@ bool OpenXrInterface::prepareXrInstance() return false; } - if (m_pimpl->htc_facial_tracking_extension_supported) + if (m_pimpl->use_expressions) { result = xrGetInstanceProcAddr(m_pimpl->instance, "xrCreateFacialTrackerHTC", (PFN_xrVoidFunction*)&(m_pimpl->pfn_xrCreateFacialTrackerHTC)); @@ -375,7 +377,7 @@ void OpenXrInterface::checkSystemProperties() facial_tracking_props.supportEyeFacialTracking = XR_FALSE; facial_tracking_props.supportLipFacialTracking = XR_FALSE; - if (m_pimpl->htc_facial_tracking_extension_supported) + if (m_pimpl->use_expressions) { *next_chain = &facial_tracking_props; next_chain = &facial_tracking_props.next; @@ -409,7 +411,7 @@ void OpenXrInterface::checkSystemProperties() yCInfo(OPENXRHEADSET, "\tOrientation Tracking: %d", system_props.trackingProperties.orientationTracking); yCInfo(OPENXRHEADSET, "\tPosition Tracking : %d", system_props.trackingProperties.positionTracking); - if (m_pimpl->htc_facial_tracking_extension_supported) + if (m_pimpl->use_expressions) { yCInfo(OPENXRHEADSET, "Facial tracking properties for system %lu: Eye tracking %d, Lip tracking %d", system_props.systemId, facial_tracking_props.supportEyeFacialTracking, facial_tracking_props.supportLipFacialTracking); @@ -1635,6 +1637,7 @@ bool OpenXrInterface::initialize(const OpenXrInterfaceSettings &settings) m_pimpl->hideWindow = settings.hideWindow; m_pimpl->renderInPlaySpace = settings.renderInPlaySpace; m_pimpl->use_gaze = settings.useGaze; + m_pimpl->use_expressions = settings.useExpressions; #ifdef DEBUG_RENDERING_LOCATION m_pimpl->renderInPlaySpace = true; diff --git a/src/devices/openxrheadset/OpenXrInterface.h b/src/devices/openxrheadset/OpenXrInterface.h index a7a615b..dd9a4bb 100644 --- a/src/devices/openxrheadset/OpenXrInterface.h +++ b/src/devices/openxrheadset/OpenXrInterface.h @@ -73,6 +73,7 @@ struct OpenXrInterfaceSettings bool hideWindow{false}; bool renderInPlaySpace{false}; bool useGaze{ true }; + bool useExpressions{ true }; }; class OpenXrInterface diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h index 412f8b1..a18153a 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h @@ -304,9 +304,6 @@ class OpenXrInterface::Implementation // flag to check if the HTC VIVE Focus3 controllers are supported by the runtime. bool focus3_supported = false; - //flag to check if the HTC facial tracking is supported by the runtime. - bool htc_facial_tracking_extension_supported = false; - //flag to check if facial tracking is supported by the headset. bool htc_eye_facial_tracking_supported = false; @@ -330,6 +327,9 @@ class OpenXrInterface::Implementation std::vector htc_eye_expressions; std::vector htc_lip_expressions; + // Flag to enable the use of expressions + bool use_expressions = true; + // Flag to enable the use of gaze bool use_gaze = true; From 01bd69c37276d28745593bc089a38721be3b0ff2 Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 16 Sep 2024 14:48:34 +0200 Subject: [PATCH 12/13] Publishing the gaze direction insead of the position --- src/devices/openxrheadset/ExpressionsManager.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/devices/openxrheadset/ExpressionsManager.cpp b/src/devices/openxrheadset/ExpressionsManager.cpp index 09edecf..c5c6270 100644 --- a/src/devices/openxrheadset/ExpressionsManager.cpp +++ b/src/devices/openxrheadset/ExpressionsManager.cpp @@ -76,13 +76,13 @@ void ExpressionsManager::setGaze(const OpenXrInterface::Pose& headPose, const Op return; } - Eigen::Vector3f gazeInHead = headPose.rotation.inverse() * (gaze.position - headPose.position); + Eigen::Vector3f gazeDirectionInHead = headPose.rotation.inverse() * gaze.rotation * Eigen::Vector3f::UnitZ(); yarp::sig::Vector& gazeVector = m_gazePort.prepare(); gazeVector.resize(3); - gazeVector[0] = gazeInHead.x(); - gazeVector[1] = gazeInHead.y(); - gazeVector[2] = gazeInHead.z(); + gazeVector[0] = gazeDirectionInHead.x(); + gazeVector[1] = gazeDirectionInHead.y(); + gazeVector[2] = gazeDirectionInHead.z(); m_gazePort.write(); } From fb52e5475fa0e0b46c242b6c965390f0033d6812 Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 25 Oct 2024 16:36:16 +0200 Subject: [PATCH 13/13] Using uppercase enum class to make ubuntu happy --- src/devices/openxrheadset/FilteredPosePublisher.cpp | 2 +- src/devices/openxrheadset/OpenXrInterface.cpp | 2 +- src/devices/openxrheadset/OpenXrInterface.h | 4 ++-- src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/devices/openxrheadset/FilteredPosePublisher.cpp b/src/devices/openxrheadset/FilteredPosePublisher.cpp index 116021f..3e55efc 100644 --- a/src/devices/openxrheadset/FilteredPosePublisher.cpp +++ b/src/devices/openxrheadset/FilteredPosePublisher.cpp @@ -148,7 +148,7 @@ void FilteredPosePublisher::updateInputPose(const OpenXrInterface::NamedPoseVelo } switch (input.filterType) { - case PoseFilterType::JumpFilter: + case PoseFilterType::JUMP_FILTER: PosePublisher::updateInputPose(filterJumps(input)); break; default: diff --git a/src/devices/openxrheadset/OpenXrInterface.cpp b/src/devices/openxrheadset/OpenXrInterface.cpp index a209a06..21b6dca 100644 --- a/src/devices/openxrheadset/OpenXrInterface.cpp +++ b/src/devices/openxrheadset/OpenXrInterface.cpp @@ -926,7 +926,7 @@ bool OpenXrInterface::prepareXrActions() gazeInputs.poses = { - {"/input/gaze_ext/pose", "pose", PoseFilterType::None} + {"/input/gaze_ext/pose", "pose", PoseFilterType::NONE} }; TopLevelPathDeclaration gaze; diff --git a/src/devices/openxrheadset/OpenXrInterface.h b/src/devices/openxrheadset/OpenXrInterface.h index dd9a4bb..d29d321 100644 --- a/src/devices/openxrheadset/OpenXrInterface.h +++ b/src/devices/openxrheadset/OpenXrInterface.h @@ -63,8 +63,8 @@ class IOpenXrQuadLayer enum class PoseFilterType { - None, - JumpFilter, + NONE, + JUMP_FILTER }; struct OpenXrInterfaceSettings diff --git a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h index a18153a..63ebcdb 100644 --- a/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h +++ b/src/devices/openxrheadset/impl/OpenXrInterfaceImpl.h @@ -116,7 +116,7 @@ struct ActionDeclaration struct PoseActionDeclaration : public ActionDeclaration { - PoseFilterType filterType{ PoseFilterType::JumpFilter }; + PoseFilterType filterType{ PoseFilterType::JUMP_FILTER }; }; struct InputActionsDeclaration