diff --git a/common/Utilities/FpsCounter.cs b/common/Utilities/FpsCounter.cs new file mode 100644 index 0000000..cd57632 --- /dev/null +++ b/common/Utilities/FpsCounter.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; + +namespace Sst.Utilities; + +public class FpsCounter { + public TimeSpan WindowSize; + + private Queue _times = new(); + + public FpsCounter(TimeSpan? windowSize = null) { + WindowSize = windowSize ?? TimeSpan.FromSeconds(0.2f); + } + + public void OnFrame() { + RemoveOutOfWindowTimes(); + _times.Enqueue(DateTime.Now); + } + + public double Read() { + RemoveOutOfWindowTimes(); + return (double)_times.Count / WindowSize.TotalSeconds; + } + + private void RemoveOutOfWindowTimes() { + var windowStart = DateTime.Now - WindowSize; + while (_times.Count > 0 && _times.Peek() < windowStart) { + _times.Dequeue(); + } + } +} diff --git a/projects/Bonelab/ColliderScope/Project.csproj b/projects/Bonelab/ColliderScope/Project.csproj index 482f733..742b80e 100644 --- a/projects/Bonelab/ColliderScope/Project.csproj +++ b/projects/Bonelab/ColliderScope/Project.csproj @@ -10,7 +10,7 @@ - + true {EAE1410F-B5CF-47D6-8764-2FCAEE822C9D} diff --git a/projects/Bonelab/ColliderScope/src/Mod.cs b/projects/Bonelab/ColliderScope/src/Mod.cs index 4ffd51f..e165256 100644 --- a/projects/Bonelab/ColliderScope/src/Mod.cs +++ b/projects/Bonelab/ColliderScope/src/Mod.cs @@ -59,14 +59,18 @@ public override void OnInitializeMelon() { "Hides head colliders so that they do not obscure your vision"); _prefOnlyResizeRigColliders = category.CreateEntry( "onlyResizeRigColliders", true, "Only resize rig colliders", - "Improves performance by watching for changes to collider size on only the rig"); + "Improves performance by watching for changes to collider size on " + + "only the rig"); // TODO: Change to frame time allocated for colliders _prefIterationsPerFrame = category.CreateEntry( "iterationsPerFrame", 8f, "Iterations per frame", - "Number of game objects to show colliders of per frame on load (higher loads faster but too high lags and crashes the game)"); + "Number of game objects to show colliders of per frame on load " + + "(higher loads faster but too high lags and crashes the game)"); _prefBackgroundIterationsPerFrame = category.CreateEntry( "backgroundIterationsPerFrame", 2f, "Background iterations per frame", - "Number of game objects to show colliders of per frame in the background (runs continuously to catch any objects added during play)"); + "Number of game objects to show colliders of per frame in the " + + "background (runs continuously to catch any objects added during " + + "play)"); LevelHooks.OnLoad += nextLevel => ResetState(); LevelHooks.OnLevelStart += level => Visualize(false); @@ -80,6 +84,7 @@ public override void OnUpdate() { if (LevelHooks.IsLoading) return; +// TODO: Replace with a check for third person camera #if DEBUG if (LevelHooks.RigManager?.ControllerRig.rightController .GetThumbStickDown() ?? diff --git a/projects/Bonelab/HandTracking/Project.csproj b/projects/Bonelab/HandTracking/Project.csproj index faf1e47..4e8f749 100644 --- a/projects/Bonelab/HandTracking/Project.csproj +++ b/projects/Bonelab/HandTracking/Project.csproj @@ -16,7 +16,15 @@ + + + + + + + + \ No newline at end of file diff --git a/projects/Bonelab/HandTracking/README.md b/projects/Bonelab/HandTracking/README.md index ec810d2..f8d865e 100644 --- a/projects/Bonelab/HandTracking/README.md +++ b/projects/Bonelab/HandTracking/README.md @@ -1 +1,41 @@ Adds support for hand tracking. + +## Installation + +- This mod is for the **patch 4 Quest standalone** version of the game +- Install **Melon Loader 0.5.7** via Lemon Loader +- Add the hand tracking permission to the game APK using Quest Patcher + - **IMPORTANT: This will RESET YOUR GAME and you will LOSE YOUR SAVE AND MODS so back them up first** + - Go to the "Tools & Options" tab -> click the "Change App" button -> select `com.StressLevelZero.BONELAB` + - Go to the "Patching" tab -> **select `None` for the mod loader** (you should have already installed Melon Loader) + - Click "Patching Options" -> **scroll to "Hand Tracking Type" -> select `V2`** + - Click the button to patch the game +- Copy this mod to `/sdcard/Android/data/com.StressLevelZero.BONELAB/files/Mods/*.dll` +- Make sure hand tracking is enabled in your Quest settings + - You should know by putting your controllers down while in the Quest menu and it switches to hands within a few seconds +- Start the game + +## Usage + +- Click on UIs by pinching +- Open the in-game menu by doing the standard Quest hand tracking menu gesture (left hand open, palm towards your face then touch the tips of your index finger and thumb) +- Walk by moving your hands up and down alternately in a running motion +- Grab by curling middle, ring and pinky fingers into a fist (same effect as pressing trigger and grip on controller) +- Force pull objects by forming a fist with these three fingers then flicking your wrist (in any direction) +- Held items like guns are triggered by curling the index finger + +## Tips + +Hand tracking's accuracy is _very_ limited. Don't expect everything to work perfectly. I did put in effort to make it less frustrating by taking into account the tracking confidence and assuming certain things when the hands are out of view, but it won't always do what you want if the headset can't see your hands/fingers. Here are some tips to help make your experience smoother: + +- When running, make sure the headset has a clear view of your hands by either: + - Holding your hands a bit higher and further forwards than normal (so they are not too close to the headset) + - Looking downwards +- When running, instead of pointing your hands forwards in a fist, open them and have your palms facing you +- Don't hold guns with two hands while aiming (headset gets confused and will make your in-game hands do weird things because your IRL hands are overlapping) +- Don't hold guns too close to your face while aiming (hand will lose tracking if too close to headset) +- While throwing things, climbing or any other action, try not to bring your hands too close to the headset or out of your eye sight + +## Fun facts + +SLZ has already added a bunch of hand tracking code. By just adding the hand tracking permission to the APK, `MarrowGame.xr.HandLeft/Right` will start tracking the hand position! Finger poses are not updated because the code is missing from the `OculusHandActionMap`, however they do actually have finger pose and gesture code in the generic `HandActionMap.ProcessesHand` method. I tried to lean on their work and get it working but there were too many gaps and it ended being easier to reimplement everything myself. diff --git a/projects/Bonelab/HandTracking/build-and-start.sh b/projects/Bonelab/HandTracking/build-and-start.sh new file mode 100755 index 0000000..f060fb9 --- /dev/null +++ b/projects/Bonelab/HandTracking/build-and-start.sh @@ -0,0 +1,22 @@ +set -eux +cd "$(dirname $0)" + +dotnet build ./Project.csproj + +# TIP: Enable wireless mode from SideQuest +adb connect 192.168.0.69 + +FILE=HandTracking.P4.ML5.dll +adb shell am force-stop com.StressLevelZero.BONELAB +adb push "./bin/Debug/Patch4_MelonLoader0.5/$FILE" /sdcard/Download/ +adb shell mv "/sdcard/Download/$FILE" /sdcard/Android/data/com.StressLevelZero.BONELAB/files/mods/ +adb shell chmod 644 "/sdcard/Android/data/com.StressLevelZero.BONELAB/files/mods/$FILE" +adb shell am start -n com.StressLevelZero.BONELAB/com.unity3d.player.UnityPlayerActivity + +# adb shell uiautomator dump /sdcard/ui_dump.xml && adb pull /sdcard/ui_dump.xml . + +# adb shell ls /sdcard/Android/data/com.StressLevelZero.BONELAB/files/melonloader +# adb pull /sdcard/Android/data/com.StressLevelZero.BONELAB/files/melonloader/etc/managed/Assembly-CSharp.dll +# adb pull /sdcard/Android/data/com.StressLevelZero.BONELAB/files/melonloader/etc/managed/SLZ.Marrow.dll + +# adb logcat -v time MelonLoader:D CRASH:D Mono:W mono:D mono-rt:D Zygote:D A64_HOOK:V DEBUG:D Binder:D AndroidRuntime:D "*:S" diff --git a/projects/Bonelab/HandTracking/src/ForcePull.cs b/projects/Bonelab/HandTracking/src/ForcePull.cs new file mode 100644 index 0000000..47b7d6e --- /dev/null +++ b/projects/Bonelab/HandTracking/src/ForcePull.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using MelonLoader; +using UnityEngine; +using SLZ.Interaction; +using Sst.Utilities; + +namespace Sst.HandTracking; + +public class ForcePull { + private const float FLICK_MAX_DURATION_SECONDS = 0.25f; + private const float FLICK_MIN_ROTATION_DEGREES = 40f; + + public HandTracker Tracker; + + private LinkedList _startingStates = new(); + private ForcePullGrip _grip; + + public bool IsPulling() => _grip?._pullToHand == Tracker.GetPhysicalHand(); + + // Run after updating Tracker.IsGripping + public void Update() { + var hand = Tracker.GetPhysicalHand(); + if (hand == null) + return; + + var hoveringGrip = + hand.farHoveringReciever?.Host.TryCast() + ?.GetForcePullGrip(); + if (hoveringGrip) { + var angle = (int)GetRotationDifference( + new Dictionary(), hoveringGrip); + Tracker.Log($"Force pull angle = {angle}"); + } + + // LogSpam( + // $"isFist={Tracker.IsGripping}, isfp={!!_grip}, + // curgo={!!hand.m_CurrentAttachedGO}, + // attrec={!!hand.AttachedReceiver}, + // farhov={!!hand.farHoveringReciever}, + // prim={hand.Controller._primaryInteractionButton}, + // sec={hand.Controller._secondaryInteractionButton}, + // grip={Tracker.ProxyController.GripButton}"); + + if (_grip?._pullToHand == hand) { + if (!Tracker.IsGripping) { + Tracker.ProxyController.GripButton = false; + _grip.CancelPull(hand); + _grip = null; + Tracker.Log("Cancelled force pull due to ungripping"); + } + return; + } + + if (_grip) { + if (hand.m_CurrentAttachedGO) { + Tracker.Log("Force pull complete"); + } else { + Tracker.Log("Cancelling force pull due to hand not being set"); + } + Tracker.ProxyController.GripButton = false; + _grip = null; + } + + var isAlreadyHolding = hand.AttachedReceiver || hand.m_CurrentAttachedGO; + if (!Tracker.IsGripping || isAlreadyHolding) + return; + + var earliest = Time.time - FLICK_MAX_DURATION_SECONDS; + while (_startingStates.Count > 0 && + _startingStates.First.Value.time < earliest) { + _startingStates.RemoveFirst(); + } + + var rotationCache = new Dictionary(); + var node = _startingStates.Last; + while (node != null) { + var handAngleDiff = Quaternion.Angle(node.Value.rotation, + Tracker.ProxyController.Rotation); + var angleFromObject = + GetRotationDifference(rotationCache, node.Value.grip); + if (handAngleDiff >= FLICK_MIN_ROTATION_DEGREES && + angleFromObject > node.Value.angleFromObject) { + // Need to set these or the force pull will instantly cancel + hand.farHoveringReciever = node.Value.reciever; + hand.Controller._primaryInteractionButton = true; + hand.Controller._secondaryInteractionButton = true; + + _grip = node.Value.grip; + _grip._pullToHand = hand; + Utils.ForcePullGrip_Pull.Call(_grip, hand); + Tracker.Log( + "Force pulling, handAngleDiff:", handAngleDiff.ToString("0.0f"), + "angleFromObject:", angleFromObject.ToString("0.0f"), + "fphand:", _grip._pullToHand, + "attached:", hand.AttachedReceiver?.name); + break; + } + node = node.Previous; + } + + var hoveringForcePullGrip = + hand.farHoveringReciever?.Host.TryCast() + ?.GetForcePullGrip(); + if (hoveringForcePullGrip) { + _startingStates.AddLast(new StartingState() { + time = Time.time, + grip = hoveringForcePullGrip, + reciever = hand.farHoveringReciever, + rotation = Tracker.ProxyController.Rotation, + angleFromObject = + GetRotationDifference(rotationCache, hoveringForcePullGrip), + }); + } + } + + private float GetRotationDifference(Dictionary cache, + ForcePullGrip grip) { + if (cache.TryGetValue(grip, out var cached)) + return cached; + // TODO: Get IRL hand position/rotation in game world (not physics rig hand) + // TODO: Is this it? (verify on pc) + var controller = Utils.RigControllerOf(Tracker).transform; + var direction = grip.transform.position - controller.position; + var target = Quaternion.LookRotation(direction); + var diff = Quaternion.Angle(target, controller.rotation); + cache.Add(grip, diff); + return diff; + } + + private struct StartingState { + public float time; + public ForcePullGrip grip; + public HandReciever reciever; + public Quaternion rotation; + public float angleFromObject; + } +} diff --git a/projects/Bonelab/HandTracking/src/HandState.cs b/projects/Bonelab/HandTracking/src/HandState.cs new file mode 100644 index 0000000..f932818 --- /dev/null +++ b/projects/Bonelab/HandTracking/src/HandState.cs @@ -0,0 +1,103 @@ +using System; +using MelonLoader; +using UnityEngine; + +namespace Sst.HandTracking; + +public class HandState { + private static Vector3 FromFlippedXVector3f(OVRPlugin.Vector3f vector) => + new Vector3(-vector.x, vector.y, vector.z); + + private static Vector3 FromFlippedZVector3f(OVRPlugin.Vector3f vector) => + new Vector3(vector.x, vector.y, -vector.z); + + private static Quaternion FromFlippedXQuatf(OVRPlugin.Quatf quat) => + new Quaternion(quat.x, -quat.y, -quat.z, quat.w); + + private static Quaternion FromFlippedZQuatf(OVRPlugin.Quatf quat) => + new Quaternion(-quat.x, -quat.y, quat.z, quat.w); + + public bool IsLeft; + public Vector3 Position; + public Quaternion Rotation; + public float Scale; + public JointTransform[] Joints = + new JointTransform[(int)OVRPlugin.SkeletonConstants.MaxHandBones]; + public OVRPlugin.TrackingConfidence HandConfidence; + public OVRPlugin.TrackingConfidence[] FingerConfidences; + public bool IsPinching; + + private OVRPlugin.Hand _hand; + private OVRPlugin.SkeletonType _skeletonType; + private OVRInput.Controller _controller; + private OVRPlugin.HandState _state = new(); + private OVRPlugin.Skeleton2 _skeleton = new(); + + public HandState(bool isLeft) { + IsLeft = isLeft; + _controller = + isLeft ? OVRInput.Controller.LHand : OVRInput.Controller.RHand; + + _skeletonType = isLeft ? OVRPlugin.SkeletonType.HandLeft + : OVRPlugin.SkeletonType.HandRight; + if (!OVRPlugin.GetSkeleton2(_skeletonType, _skeleton)) { + throw new Exception("Failed to get hand skeleton"); + } + + _hand = isLeft ? OVRPlugin.Hand.HandLeft : OVRPlugin.Hand.HandRight; + Update(); + } + + public void Update() { + if (!OVRPlugin.GetHandState(OVRPlugin.Step.Render, _hand, _state)) { + throw new Exception("Failed to get hand state"); + } + + Position = FromFlippedZVector3f(_state.RootPose.Position); + Rotation = FromFlippedZQuatf(_state.RootPose.Orientation); + Scale = _state.HandScale; + + for (var i = 0; i < Joints.Length; i++) { + var localRot = FromFlippedXQuatf(_state.BoneRotations[i]); + var parentIdx = _skeleton.Bones[i].ParentBoneIndex; + var parentJoint = + OVRPlugin.IsValidBone((OVRPlugin.BoneId)parentIdx, _skeletonType) + // parentIdx will always be less than i so Joints[parentIdx] will + // already be updated + ? Joints[parentIdx] + : JointTransform.IDENTITY; + var handRot = parentJoint.HandRotation * localRot; + var localPos = + handRot * FromFlippedXVector3f(_skeleton.Bones[i].Pose.Position); + var handPos = parentJoint.HandPosition + localPos; + + Joints[i] = new JointTransform() { + LocalPosition = localPos, + LocalRotation = localRot, + HandPosition = handPos, + HandRotation = handRot, + }; + } + + HandConfidence = _state.HandConfidence; + FingerConfidences = _state.FingerConfidences; + + IsPinching = (_state.Pinches & OVRPlugin.HandFingerPinch.Index) != 0; + } + + public bool IsActive() => + // NOTE: Requires OVRInput.Update() to be called (game already does this) + OVRInput.IsControllerConnected(_controller); + + public bool IsTracked() => (_state.Status & + OVRPlugin.HandStatus.HandTracked) != 0; +} + +public struct JointTransform { + public static JointTransform IDENTITY; + + public Vector3 LocalPosition; + public Quaternion LocalRotation; + public Vector3 HandPosition; + public Quaternion HandRotation; +} diff --git a/projects/Bonelab/HandTracking/src/HandTracker.cs b/projects/Bonelab/HandTracking/src/HandTracker.cs new file mode 100644 index 0000000..14132d0 --- /dev/null +++ b/projects/Bonelab/HandTracking/src/HandTracker.cs @@ -0,0 +1,427 @@ +using System; +using System.Linq; +using System.Collections.Generic; +using MelonLoader; +using UnityEngine; +using SLZ.Marrow.Input; +using SLZ.Rig; +using SLZ.Interaction; +using Sst.Utilities; +using SLZ.Bonelab; + +namespace Sst.HandTracking; + +public class HandTracker { + public struct Options { + public bool isLeft; + public XRController marrowController; + public Action setMarrowController; + public OVRInput.Controller ovrController; + public OVRInput.Controller ovrHand; + // Rotation is off for some reason so we need to correct it + // TODO: Are these offsets related to HandActionMap.LeftAnimSpace? + // TODO: Seem to be different offsets per avatar + public Quaternion handRotationOffset; + public Vector3 handPositionOffset; + } + + private const float GRIP_CURL_THRESHOLD = 0.6f; + private const float TRIGGER_DIST_FROM_WRIST = 0.1f; + private const float MAX_HOVER_LOCK_TIME = 0.5f; + private static OVRPlugin.BoneId[] FINGER_JOINTS_THUMB = { + OVRPlugin.BoneId.Hand_Thumb0, + OVRPlugin.BoneId.Hand_Thumb1, + OVRPlugin.BoneId.Hand_Thumb2, + OVRPlugin.BoneId.Hand_Thumb3, + }; + private static OVRPlugin.BoneId[] FINGER_JOINTS_INDEX = { + OVRPlugin.BoneId.Hand_Index1, + OVRPlugin.BoneId.Hand_Index2, + OVRPlugin.BoneId.Hand_Index3, + }; + private static OVRPlugin.BoneId[] FINGER_JOINTS_MIDDLE = { + OVRPlugin.BoneId.Hand_Middle1, + OVRPlugin.BoneId.Hand_Middle2, + OVRPlugin.BoneId.Hand_Middle3, + }; + private static OVRPlugin.BoneId[] FINGER_JOINTS_RING = { + OVRPlugin.BoneId.Hand_Ring1, + OVRPlugin.BoneId.Hand_Ring2, + OVRPlugin.BoneId.Hand_Ring3, + }; + private static OVRPlugin.BoneId[] FINGER_JOINTS_PINKY = { + OVRPlugin.BoneId.Hand_Pinky1, + OVRPlugin.BoneId.Hand_Pinky2, + OVRPlugin.BoneId.Hand_Pinky3, + }; + private static OVRPlugin.HandFinger[] GRIP_FINGERS = { + OVRPlugin.HandFinger.Middle, + OVRPlugin.HandFinger.Ring, + OVRPlugin.HandFinger.Pinky, + }; + + public Options Opts; + public bool IsTracking = false; + public bool PinchUp = false; + public bool IsPinching = false; + public bool IsMenuOpen = false; + public bool IsGripping = false; + public bool Proxy = false; + public XRController ProxyController; + public LocoState LocoState; + public HandState HandState; + + private OVRPlugin.HandState _handState; + private OVRPlugin.Skeleton2 _skeleton; + private float[] _fingerCurls = new float[(int)OVRPlugin.HandFinger.Max]; + private bool[] _fingerGripStates = new bool[(int)OVRPlugin.HandFinger.Max]; + private UIControllerInput _uiControllerInput; + private ForcePull _forcePull; + private Hand _physicalHand; + + private int _logIndex = 0; + internal void Log(params object[] messageParts) { +#if DEBUG + Mod.Instance.LoggerInstance.Msg( + (Opts.isLeft ? "[L] " : "[R] ") + + string.Join(" ", messageParts.Select(part => part.ToString()))); +#endif + } + internal void LogSpam(params object[] messageParts) { +#if DEBUG + if (_logIndex % 100 == 0) + Log(messageParts); +#endif + } + + public HandTracker(Options options) { + Opts = options; + + ProxyController = new ControllerActionMap() { + _DeviceInfo_k__BackingField = "ProxyController", + _xrDevice = new() { m_Initialized = true }, + Type = XRControllerType.OculusTouch, + }; + HandState = new(Opts.isLeft); + + _forcePull = new() { Tracker = this }; + + Log("Initialized HandTracker"); + } + + public bool + IsControllerConnected() => OVRInput.IsControllerConnected(Opts.ovrController); + + public void OnUpdate() { + _logIndex++; + + if (IsControllerConnected()) { + if (IsTracking) { + IsTracking = false; + Opts.setMarrowController(Opts.marrowController); + Log("Hand tracking is now inactive"); + } + return; + } + + if (!HandState.IsActive()) + return; + + if (!IsTracking) { + IsTracking = true; + Opts.setMarrowController(ProxyController); + Log("Hand tracking is now active"); + } + } + + public void UpdateProxyController() { + HandState.Update(); + + ProxyController._IsConnected_k__BackingField = true; + ProxyController.IsTracking = HandState.IsTracked(); + if (!ProxyController.IsTracking) + return; + + ProxyController.Rotation = HandState.Rotation * Opts.handRotationOffset; + ProxyController.Position = + HandState.Position + ProxyController.Rotation * Opts.handPositionOffset; + + var ovrHand = + Opts.isLeft ? OVRPlugin.Hand.HandLeft : OVRPlugin.Hand.HandRight; + if (_handState == null) + _handState = new OVRPlugin.HandState(); + // TODO: Try out wide motion mode + if (!OVRPlugin.GetHandState(OVRPlugin.Step.Render, ovrHand, _handState)) { + _handState = null; + return; + } + + if (_skeleton == null) { + _skeleton = new OVRPlugin.Skeleton2(); + var skeletonType = Opts.isLeft ? OVRPlugin.SkeletonType.HandLeft + : OVRPlugin.SkeletonType.HandRight; + if (!OVRPlugin.GetSkeleton2(skeletonType, _skeleton)) { + _skeleton = null; + MelonLogger.Warning("Failed to get hand skeleton"); + return; + } + } + + UpdateFingerCurls(); + UpdateLocomotion(); + UpdateUiPinch(); + _forcePull.Update(); + UpdateTrigger(); + UpdateMenu(); + // TODO: Jumping + // TODO: Inventory + } + + private void UpdateFingerCurls() { + _fingerCurls[(int)OVRPlugin.HandFinger.Thumb] = + CalculateFingerCurl(FINGER_JOINTS_THUMB, 100f); + _fingerCurls[(int)OVRPlugin.HandFinger.Index] = + CalculateFingerCurl(FINGER_JOINTS_INDEX, 200f); + _fingerCurls[(int)OVRPlugin.HandFinger.Middle] = + CalculateFingerCurl(FINGER_JOINTS_MIDDLE, 200f); + _fingerCurls[(int)OVRPlugin.HandFinger.Ring] = + CalculateFingerCurl(FINGER_JOINTS_RING, 200f); + _fingerCurls[(int)OVRPlugin.HandFinger.Pinky] = + CalculateFingerCurl(FINGER_JOINTS_PINKY, 200f); + + ProxyController.ThumbFinger = _fingerCurls[(int)OVRPlugin.HandFinger.Thumb]; + ProxyController.IndexFinger = _fingerCurls[(int)OVRPlugin.HandFinger.Index]; + ProxyController.MiddleFinger = + _fingerCurls[(int)OVRPlugin.HandFinger.Middle]; + ProxyController.RingFinger = _fingerCurls[(int)OVRPlugin.HandFinger.Ring]; + ProxyController.PinkyFinger = _fingerCurls[(int)OVRPlugin.HandFinger.Pinky]; + + foreach (var finger in GRIP_FINGERS) { + if (_handState.FingerConfidences[(int)finger] == + OVRPlugin.TrackingConfidence.High) { + _fingerGripStates[(int)finger] = + _fingerCurls[(int)finger] >= GRIP_CURL_THRESHOLD; + } + } + IsGripping = GRIP_FINGERS.All(finger => _fingerGripStates[(int)finger]); + + if (IsGripping) { + ProxyController.GripButtonDown = !ProxyController.GripButton; + ProxyController.GripButtonUp = false; + ProxyController.GripButton = true; + ProxyController.Grip = 1f; + if (ProxyController.GripButtonDown) + Log("Grip button down"); + } else { + ProxyController.GripButtonUp = ProxyController.GripButton; + ProxyController.GripButtonDown = false; + ProxyController.GripButton = false; + ProxyController.Grip = 0f; + if (ProxyController.GripButtonUp) + Log("Grip button up"); + } + } + + private float CalculateFingerCurl(OVRPlugin.BoneId[] fingerJoints, + float maxRotation) { + var totalRotation = 0f; + var prevJointRot = ToQuaternion( + _handState.BoneRotations[(int)OVRPlugin.BoneId.Hand_WristRoot]); + for (var i = 0; i < fingerJoints.Length; i++) { + var jointRotation = + ToQuaternion(_handState.BoneRotations[(int)fingerJoints[i]]); + var relativeRotation = Quaternion.Inverse(prevJointRot) * jointRotation; + totalRotation += (relativeRotation.eulerAngles.z + 180f) % 360f - 180f; + } + + return MapToFingerCurve(Mathf.Clamp01(totalRotation / maxRotation)); + } + + // Correct for Bonelab controller finger not curling linearly + private float MapToFingerCurve(float linearCurl) { + var mapping = new(float, float)[] { + (0.0f, 0.00f), (0.1f, 0.10f), (0.2f, 0.13f), (0.3f, 0.20f), + (0.4f, 0.32f), (0.5f, 0.38f), (0.6f, 0.42f), (0.7f, 0.52f), + (0.8f, 0.68f), (0.9f, 0.84f), (1.0f, 1.00f), + }; + + for (int i = 0; i < mapping.Length - 1; i++) { + var inputMin = mapping[i].Item1; + var inputMax = mapping[i + 1].Item1; + if (linearCurl >= inputMin && linearCurl <= inputMax) { + var t = (linearCurl - inputMin) / (inputMax - inputMin); + var outputMin = mapping[i].Item2; + var outputMax = mapping[i + 1].Item2; + return outputMin + t * (outputMax - outputMin); + } + } + + return linearCurl; + } + + // TODO: Replace with utils + private Quaternion ToQuaternion(OVRPlugin.Quatf quatf) => + new Quaternion(quatf.x, quatf.y, quatf.z, quatf.w); + + private void UpdateLocomotion() { + if (Opts.isLeft == Utils.IsLocoControllerLeft() && + LocoState.Axis.HasValue) { + ProxyController.Joystick2DAxis = LocoState.Axis.Value; + + if (LocoState.Axis.Value.sqrMagnitude > 0.1f) { + ProxyController.JoystickButtonDown = !ProxyController.JoystickButton; + ProxyController.JoystickButtonUp = false; + ProxyController.JoystickButton = true; + ProxyController.JoystickTouch = true; + } else { + ProxyController.JoystickButtonUp = ProxyController.JoystickButton; + ProxyController.JoystickButtonDown = false; + ProxyController.JoystickButton = false; + ProxyController.JoystickTouch = false; + } + } + } + + private void UpdateUiPinch() { + var isPinching = + (_handState.Pinches & OVRPlugin.HandFingerPinch.Index) != 0; + + if (ProxyController.BButton && PinchUp) { + IsMenuOpen = false; + Log("Closed menu due to pinch"); + } + + PinchUp = isPinching && !IsPinching; + IsPinching = isPinching; + } + + private void UpdateTrigger() { + if (IsTriggerPressed()) { + ProxyController.TriggerButtonDown = !ProxyController.TriggerButton; + ProxyController.TriggerButtonUp = false; + ProxyController.TriggerButton = true; + ProxyController.TriggerTouched = true; + ProxyController.Trigger = 1f; + if (ProxyController.TriggerButtonDown) + Log("Trigger button down"); + } else { + ProxyController.TriggerButtonUp = ProxyController.TriggerButton; + ProxyController.TriggerButtonDown = false; + ProxyController.TriggerButton = false; + ProxyController.TriggerTouched = false; + ProxyController.Trigger = 0f; + if (ProxyController.TriggerButtonUp) + Log("Trigger button up"); + } + } + + private bool IsTriggerPressed() { + var physicsRig = LevelHooks.RigManager?.physicsRig; + if (physicsRig == null) + return false; + + var hand = Opts.isLeft ? physicsRig.leftHand : physicsRig.rightHand; + if (_forcePull.IsPulling()) + return true; + + // TODO: Are there any triggerable objects which don't use TargetGrip? + var isHoldingObject = hand.AttachedReceiver?.TryCast() != null; + if (!isHoldingObject) + return IsGripping; + + var indexPos = GetRelativeIndexTipPos(); + return indexPos.x > 0f; + } + + private Vector3 GetRelativeIndexTipPos() { + var jointPos = Vector3.zero; + var jointRot = Quaternion.identity; + // TODO: Can we work backwards from the tip based on parent bone? + foreach (var boneId in new[] { + OVRPlugin.BoneId.Hand_Index1, + OVRPlugin.BoneId.Hand_Index2, + OVRPlugin.BoneId.Hand_Index3, + OVRPlugin.BoneId.Hand_IndexTip, + }) { + var rot = _handState.BoneRotations[(int)boneId]; + jointRot *= + Utils.FromFlippedXQuatf(_handState.BoneRotations[(int)boneId]); + jointPos += jointRot * Utils.FromFlippedXVector3f( + _skeleton.Bones[(int)boneId].Pose.Position); + } + return Opts.isLeft ? jointPos : -jointPos; + } + + private void UpdateMenu() { + var menu = UIRig.Instance?.popUpMenu; + if (!menu) + return; + var rigManager = LevelHooks.RigManager; + if (!rigManager) + return; + var rigController = Utils.RigControllerOf(this); + + var isMenuPressed = + (_handState.Status & OVRPlugin.HandStatus.MenuPressed) != 0; + if (isMenuPressed) { + if (menu.m_IsActivated) { + menu.Deactivate(); + } else { + var physicsRig = rigManager.physicsRig; + menu.Activate(rigManager.ControllerRig.m_head, physicsRig.m_chest, + GetUiControllerInput(rigController), rigController); + } + return; + } + + if (menu.m_IsActivated) { + if (PinchUp) { + menu.Trigger(false, false, GetUiControllerInput(rigController)); + } + } + + // TODO: Do I need this for things like changing constrainer mode? + // if ((_handState.Status & OVRPlugin.HandStatus.MenuPressed) != 0) { + // ProxyController.BButtonDown = !ProxyController.BButton; + // ProxyController.BButtonUp = false; + // ProxyController.BButton = true; + // if (ProxyController.BButtonDown) + // Log("B pressed via menu gesture"); + // } else { + // ProxyController.BButtonUp = ProxyController.BButton; + // ProxyController.BButtonDown = false; + // ProxyController.BButton = false; + // if (ProxyController.BButtonUp) + // Log("B up"); + // } + } + + private UIControllerInput GetUiControllerInput(BaseController rigController) { + if (!_uiControllerInput) { + _uiControllerInput = rigController.GetComponent(); + } + return _uiControllerInput; + } + + public Hand GetPhysicalHand() { + if (_physicalHand) + return _physicalHand; + + var physicsRig = LevelHooks.RigManager?.physicsRig; + if (physicsRig == null) + return null; + + _physicalHand = Opts.isLeft ? physicsRig.leftHand : physicsRig.rightHand; + return _physicalHand; + } + + // TODO: Animate fingers to the tracked joint positions instead of using curl + public void OnOpenControllerProcessFingers(OpenController openController) { + // Skip processing our curl values because they are already good + openController._processedThumb = ProxyController.ThumbFinger; + openController._processedIndex = ProxyController.IndexFinger; + openController._processedMiddle = ProxyController.MiddleFinger; + openController._processedRing = ProxyController.RingFinger; + openController._processedPinky = ProxyController.PinkyFinger; + } +} diff --git a/projects/Bonelab/HandTracking/src/Locomotion.cs b/projects/Bonelab/HandTracking/src/Locomotion.cs new file mode 100644 index 0000000..d118800 --- /dev/null +++ b/projects/Bonelab/HandTracking/src/Locomotion.cs @@ -0,0 +1,124 @@ +using System; +using System.Linq; +using MelonLoader; +using UnityEngine; + +namespace Sst.HandTracking; + +public class LocoState { + private const float CONFIDENCE_BUILD_RATE = 4f; + private const float CONFIDENCE_DRAIN_RATE = 1f; + private const float VELOCITY_MIN = 1f; + private const float VELOCITY_MAX = 2f; + private const float VELOCITY_FACTOR = 1f / (VELOCITY_MAX - VELOCITY_MIN); + private const float DIVERGENCE_MIN = 0.4f; + private const float DIVERGENCE_MAX = 1.4f; + private const float DIVERGENCE_FACTOR = + 1f / (DIVERGENCE_MAX - DIVERGENCE_MIN); + private const float HEIGHT_MIN = 0.2f; + private const float HEIGHT_MAX = 0.5f; + private const float HEIGHT_FACTOR = 1f / (HEIGHT_MAX - HEIGHT_MIN); + + public Vector2? Axis; + + private LocoHandState _left; + private LocoHandState _right; + private float _confidence = 0f; + + public void Init(HandTracker tracker) { + var state = new LocoHandState() { Tracker = tracker }; + if (tracker.Opts.isLeft) { + _left = state; + } else { + _right = state; + } + tracker.LocoState = this; + } + + public void Update() { + // Apparently the game just swaps the MarrowGame.xr controllers when the + // left handed setting is enabled so the left tracker always has the + // locomotion stick + var locoTracker = _left.Tracker; + if (locoTracker.IsControllerConnected() || _left == null || + _right == null) { + Axis = null; + return; + } + + _left.Update(); + _right.Update(); + + var (stateMax, stateMin) = + Mathf.Abs(_left.Velocity) > Mathf.Abs(_right.Velocity) + ? (_left, _right) + : (_right, _left); + var scoreMaxVelocity = + (Mathf.Abs(stateMax.Velocity) - VELOCITY_MIN) * VELOCITY_FACTOR; + var scoreCorrespondence = Mathf.Clamp01( + (DIVERGENCE_MAX - Mathf.Abs(stateMax.Velocity + stateMin.Velocity)) * + DIVERGENCE_FACTOR); + var scoreSwing = Mathf.Clamp01( + (Mathf.Max(_left.SwingSize, _right.SwingSize) - HEIGHT_MIN) * + HEIGHT_FACTOR); + + // TODO: Allow one handed running after building confidence + var confidenceChangeRate = scoreMaxVelocity * scoreCorrespondence * + scoreSwing * CONFIDENCE_BUILD_RATE - + CONFIDENCE_DRAIN_RATE; + _confidence = Mathf.Clamp( + _confidence + confidenceChangeRate * Time.deltaTime, 0f, 1.2f); + + // Dbg.Log(string.Join( + // ", ", + // new[] { + // ("c", _confidence), + // ("sts", stateMax.SwingSize), + // ("stv", stateMax.Velocity), + // ("smv", scoreMaxVelocity), + // ("sc", scoreCorrespondence), + // ("ss", scoreSwing), + // } + // .Select(x => $"{x.Item1}={x.Item2.ToString(" 0.00;-0.00")}"))); + + Axis = new Vector2(0f, Mathf.Clamp01(_confidence)); + } +} + +public class LocoHandState { + public HandTracker Tracker; + public float Velocity; + public float SwingSize; + + private bool _isTracked; + private float _predictedNextHeight; + public float _height; + private float _minHeight; + private float _maxHeight; + + // TODO: Continue running with low confidence hand states + public void Update() { + _isTracked = + Tracker.IsControllerConnected() || Tracker.HandState.IsActive(); + if (!_isTracked || Time.deltaTime == 0f) + return; + + var prevHeight = _height; + // TODO: Convert to game units (meters if player is 1.78m tall) + _height = (Tracker.HandState.IsActive() ? Tracker.ProxyController + : Tracker.Opts.marrowController) + .Position.y; + + var prevVelocity = Velocity; + Velocity = (_height - prevHeight) / Time.deltaTime; + + if (prevVelocity <= 0f && Velocity > 0f) { + _minHeight = _predictedNextHeight; + } else if (prevVelocity >= 0f && Velocity < 0f) { + _maxHeight = _predictedNextHeight; + } + SwingSize = _maxHeight - _minHeight; + + _predictedNextHeight = _height + Velocity; + } +} diff --git a/projects/Bonelab/HandTracking/src/Mod.cs b/projects/Bonelab/HandTracking/src/Mod.cs index ba07749..0c17226 100644 --- a/projects/Bonelab/HandTracking/src/Mod.cs +++ b/projects/Bonelab/HandTracking/src/Mod.cs @@ -1,71 +1,243 @@ using System; using System.Linq; -using HarmonyLib; using MelonLoader; using UnityEngine; +using SLZ.Marrow.Utilities; using Sst.Utilities; +using SLZ.Bonelab; +using SLZ.Marrow.Warehouse; +using HarmonyLib; +using SLZ.Marrow.Input; +using SLZ.Interaction; +using SLZ.Rig; +using System.Collections.Generic; +using SLZ.Marrow.Interaction; namespace Sst.HandTracking; public class Mod : MelonMod { - public static (OVRInput.Controller, Color)[] HANDS = { - (OVRInput.Controller.LHand, Color.red), - (OVRInput.Controller.RHand, Color.blue), - }; - public static Mod Instance; - public (OVRInput.Controller, GameObject)[] Visualizations = null; + private HandTracker[] _trackers = { null, null }; + private LocoState _locoState = new LocoState(); + private FpsCounter _fpsRefresh = new(TimeSpan.FromSeconds(1f)); + private FpsCounter _fpsUpdate = new(TimeSpan.FromSeconds(1f)); + private FpsCounter _fpsFixed = new(TimeSpan.FromSeconds(1f)); + private HashSet _visibleLaserCursors = new(); + + private HandTracker _trackerLeft { + get => _trackers[0]; + set => _trackers[0] = value; + } + private HandTracker _trackerRight { + get => _trackers[1]; + set => _trackers[1] = value; + } public override void OnInitializeMelon() { Dbg.Init(BuildInfo.NAME); Instance = this; - // UnityEngine.XR.Hand.Hand_TryGetFingerBonesAsList( - // 1, UnityEngine.XR.HandFinger.Index, out var rootBone); + LevelHooks.OnLoad += nextLevel => _visibleLaserCursors.Clear(); } public override void OnUpdate() { - var parent = LevelHooks.RigManager?.ControllerRig.transform ?? - LevelHooks.BasicTrackingRig?.transform; - if (!parent) + _fpsUpdate.OnFrame(); + // Dbg.Log( + // $"fixed={_fpsFixed.Read().ToString("N1")}, + // update={_fpsUpdate.Read().ToString("N1")}, + // refresh={_fpsRefresh.Read().ToString("N1")}"); + + if (!MarrowGame.IsInitialized || MarrowGame.xr == null) return; - if (!OVRInput.IsControllerConnected(OVRInput.Controller.Hands)) { - if (Visualizations != null) { - foreach (var (hand, obj) in Visualizations) { - if (obj) - GameObject.Destroy(obj); - } - Visualizations = null; - MelonLogger.Msg("Hand tracking is now inactive"); - } + if (_trackerLeft == null && MarrowGame.xr.LeftController != null) { + _trackerLeft = new(new() { + isLeft = true, + marrowController = MarrowGame.xr.LeftController, + setMarrowController = c => MarrowGame.xr.LeftController = c, + ovrController = OVRInput.Controller.LTouch, + ovrHand = OVRInput.Controller.LHand, + handRotationOffset = Quaternion.Euler(0f, 90f, 0f) * + Quaternion.Euler(0f, 0f, 95f) * + Quaternion.Euler(345f, 0f, 0f), + handPositionOffset = new Vector3(0.04f, 0.02f, 0.1f), + }); + _locoState.Init(_trackerLeft); + } + + if (_trackerRight == null && MarrowGame.xr.RightController != null) { + _trackerRight = new(new() { + isLeft = false, + marrowController = MarrowGame.xr.RightController, + setMarrowController = c => MarrowGame.xr.RightController = c, + ovrController = OVRInput.Controller.RTouch, + ovrHand = OVRInput.Controller.RHand, + handRotationOffset = Quaternion.Euler(275f, 0f, 0f) * + Quaternion.Euler(0f, 270f, 0f) * + Quaternion.Euler(345f, 0f, 0f), + handPositionOffset = new Vector3(-0.04f, 0.02f, 0.1f), + }); + _locoState.Init(_trackerRight); + } + + // TODO: Can we do the updates right before the inputs are used? + _trackerLeft.OnUpdate(); + _trackerRight.OnUpdate(); + + // TODO: Do this on fixed update or somewhere frame rate independent + _locoState.Update(); + } + + public override void OnFixedUpdate() { _fpsFixed.OnFrame(); } + +#if DEBUG + public override void OnSceneWasInitialized(int buildindex, string sceneName) { + if (!sceneName.ToUpper().Contains("BOOTSTRAP")) return; + AssetWarehouse.OnReady(new Action(() => { + var crate = AssetWarehouse.Instance.GetCrates().ToArray().First( + c => c.Barcode.ID == Levels.Barcodes.HUB); + var bootstrapper = + GameObject.FindObjectOfType(); + var crateRef = new LevelCrateReference(crate.Barcode.ID); + bootstrapper.VoidG114CrateRef = crateRef; + bootstrapper.MenuHollowCrateRef = crateRef; + })); + } +#endif + + internal HandTracker GetTrackerFromProxyController(XRController controller) { + if (_trackerLeft.ProxyController.Equals(controller)) + return _trackerLeft; + if (_trackerRight.ProxyController.Equals(controller)) + return _trackerRight; + return null; + } + + // TODO: Is there no way to make a ProxyController class with its own + // Refresh? + [HarmonyPatch(typeof(ControllerActionMap), + nameof(ControllerActionMap.Refresh))] + internal static class ControllerActionMap_Refresh { + [HarmonyPrefix] + private static bool Prefix(ControllerActionMap __instance) { + var tracker = Mod.Instance.GetTrackerFromProxyController(__instance); + if (tracker == null || !tracker.IsTracking) + return true; + + if (tracker.Opts.isLeft) + Instance._fpsRefresh.OnFrame(); + + tracker.UpdateProxyController(); + return false; + } + } + + [HarmonyPatch(typeof(OpenController), nameof(OpenController.ProcessFingers))] + internal static class OpenController_ProcessFingers { + [HarmonyPrefix] + private static bool Prefix(OpenController __instance) { + var tracker = Mod.Instance.GetTrackerFromProxyController( + Utils.XrControllerOf(__instance)); + if (tracker == null || !tracker.IsTracking) + return true; + + tracker.OnOpenControllerProcessFingers(__instance); + return false; + } + } + + [HarmonyPatch(typeof(LaserCursor), nameof(LaserCursor.ShowCursor))] + internal static class LaserCursor_ShowCursor { + [HarmonyPostfix] + private static void Postfix(LaserCursor __instance) { + // TODO: Point laser pointer in direction of hand + if (!__instance.cursorHidden) + Mod.Instance._visibleLaserCursors.Add(__instance); + } + } + + [HarmonyPatch(typeof(LaserCursor), nameof(LaserCursor.HideCursor))] + internal static class LaserCursor_HideCursor { + [HarmonyPostfix] + private static void Postfix(LaserCursor __instance) { + if (__instance.cursorHidden) + Mod.Instance._visibleLaserCursors.Remove(__instance); } + } - if (Visualizations == null) { - Visualizations = CreateVisualizations(parent); - MelonLogger.Msg("Hand tracking is now active"); + [HarmonyPatch(typeof(LaserCursor), nameof(LaserCursor.Update))] + internal static class LaserCursor_Update { + [HarmonyPrefix] + private static void Prefix(LaserCursor __instance, ref bool __state) { + var pinchedTracker = + Mod.Instance._trackers.FirstOrDefault(t => t?.PinchUp ?? false); + if (pinchedTracker == null) + return; + + // Show laser pointer from the hand which is pinching + var controller = Utils.RigControllerOf(pinchedTracker); + Control_UI_InGameData.SetActiveController(controller); + // __instance.activeController = controller; + __instance.controllerFocused = true; + __state = true; } - UpdateHandPositions(); + [HarmonyPostfix] + private static void Postfix(LaserCursor __instance, ref bool __state) { + if (__state) + __instance.Trigger(); + } } +} + +public static class Utils { + public static bool IsLocoControllerLeft() => + UIRig.Instance?.controlPlayer?.body_vitals?.isRightHanded ?? true; + + public static Vector3 FromFlippedXVector3f(OVRPlugin.Vector3f vector) => + new Vector3(-vector.x, vector.y, vector.z); + + public static Vector3 FromFlippedZVector3f(OVRPlugin.Vector3f vector) => + new Vector3(vector.x, vector.y, -vector.z); + + public static Quaternion FromFlippedXQuatf(OVRPlugin.Quatf quat) => + new Quaternion(quat.x, -quat.y, -quat.z, quat.w); + + public static XRController XrControllerOf(BaseController controller) => + controller.handedness == Handedness.LEFT ? MarrowGame.xr.LeftController + : controller.handedness == Handedness.RIGHT + ? MarrowGame.xr.RightController + : null; + + public static BaseController RigControllerOf(HandTracker tracker) { + var controllerRig = LevelHooks.RigManager?.ControllerRig; + if (controllerRig == null) + return null; + return tracker.Opts.isLeft ? controllerRig.leftController + : controllerRig.rightController; + } + + [HarmonyPatch(typeof(ForcePullGrip), nameof(ForcePullGrip.Pull))] + internal static class ForcePullGrip_Pull { + private static bool _enableForcePull = false; + + public static void Call(ForcePullGrip grip, Hand hand) { + _enableForcePull = true; + grip.Pull(hand); + _enableForcePull = false; + } + + [HarmonyPrefix] + private static bool Prefix(ForcePullGrip __instance, Hand hand) { + if (_enableForcePull || Mod.Instance.GetTrackerFromProxyController( + XrControllerOf(hand.Controller)) == null) + return true; - private (OVRInput.Controller, - GameObject)[] CreateVisualizations(Transform parent) => - HANDS - .Select(hand => { - var obj = GameObject.CreatePrimitive(PrimitiveType.Sphere); - obj.transform.SetParent(parent); - obj.transform.localScale = new Vector3(0.1f, 0.1f, 0.1f); - obj.GetComponent().material.color = hand.Item2; - return (hand.Item1, obj); - }) - .ToArray(); - - private void UpdateHandPositions() { - foreach (var (hand, obj) in Visualizations) { - obj.transform.position = OVRInput.GetLocalControllerPosition(hand); + Dbg.Log("ForcePullGrip.Pull was called but is disabled"); + __instance._pullToHand = null; + return _enableForcePull; } } }