Quest Pro Eyetracking

Cognitive3DTest/Plugins/Cognitive3D/Source/Cognitive3D/Private/ActiveSessionView.cpp

@@ -5,11 +5,6 @@
 #include "ActiveSessionView.h"
 #include "HeadMountedDisplayFunctionLibrary.h"
 
-float AActiveSessionView::Remap(float num, float low1, float high1, float low2, float high2)
-{
-	return low2 + (num - low1) * (high2 - low2) / (high1 - low1);
-}
-
 // Sets default values
 AActiveSessionView::AActiveSessionView()
 {
@@ -69,16 +64,43 @@ void AActiveSessionView::DelaySetupWidget()
 	if (WidgetComponent != NULL)
 	{
 		auto widgetInstance = WidgetComponent->GetUserWidgetObject();
-		UHeadMountedDisplayFunctionLibrary::SetSpectatorScreenModeTexturePlusEyeLayout(FVector2D(0, 0), FVector2D(1, 1), FVector2D(0, 0), FVector2D(1, 1),true,false,true);
+		UHeadMountedDisplayFunctionLibrary::SetSpectatorScreenModeTexturePlusEyeLayout(FVector2D(0, 0), FVector2D(1, 1), FVector2D(0, 0), FVector2D(1, 1), true, false, true);
 		UTextureRenderTarget2D* widgetTextureRT2D = WidgetComponent->GetRenderTarget();
 		UHeadMountedDisplayFunctionLibrary::SetSpectatorScreenTexture((UTexture*)widgetTextureRT2D);
 		UHeadMountedDisplayFunctionLibrary::SetSpectatorScreenMode(ESpectatorScreenMode::TexturePlusEye);
 	}
+
+
+	//screen size debugging
+	//add   #include "GameFramework/GameUserSettings.h"
+	/*
+	auto provider = FAnalyticsCognitive3D::Get().GetCognitive3DProvider();
+
+	UGameUserSettings* MyGameSettings = GEngine->GetGameUserSettings();
+	FIntPoint screenResolution = MyGameSettings->GetScreenResolution();
+	provider.Pin()->SetSessionProperty("ScreenResolution", FString::FromInt(screenResolution.X) + "  " + FString::FromInt(screenResolution.Y));
+	FIntPoint desktopResolution = MyGameSettings->GetDesktopResolution();
+	provider.Pin()->SetSessionProperty("DesktopResolution", FString::FromInt(desktopResolution.X) + "  " + FString::FromInt(desktopResolution.Y));
+
+	FVector WorldPosition;
+	FVector2D ScreenPosition;
+	ULocalPlayer* const LP = PlayerController ? PlayerController->GetLocalPlayer() : nullptr;
+	if (LP && LP->ViewportClient)
+	{
+		// get the projection data
+		FSceneViewProjectionData ProjectionData;
+		if (LP->GetProjectionData(LP->ViewportClient->Viewport, ProjectionData))
+		{
+			FMatrix const ViewProjectionMatrix = ProjectionData.ComputeViewProjectionMatrix();
+			bool bResult = FSceneView::ProjectWorldToScreen(WorldPosition, ProjectionData.GetConstrainedViewRect(), ViewProjectionMatrix, ScreenPosition);
+			provider.Pin()->SetSessionProperty("ProjectionRect", FString::FromInt(ProjectionData.GetViewRect().Width()) + "  " + FString::FromInt(ProjectionData.GetViewRect().Height()));
+		}
+	}*/
 }
 
 void AActiveSessionView::Tick(float delta)
 {
-	
+
 }
 
 TArray<FVector> AActiveSessionView::GetProjectedFixations()
@@ -107,11 +129,12 @@ TArray<FVector> AActiveSessionView::GetProjectedFixations()
 		PlayerController->ProjectWorldLocationToScreen(worldPosition, screenPosition);
 
 		//remap from hmd to spectator view
-		float x = Remap(screenPosition.X, 0, 2880, 0, 1280);
-		float y = Remap(screenPosition.Y, 0, 1600, 0, 720);
+		float x = Remap(screenPosition.X, 0, GetHMDWidth(), 0, SpectatorWidth);
+		float y = Remap(screenPosition.Y, 0, GetHMDHeight(), 0, SpectatorHeight);
 
 		//slightly shift the x value (because projection is off center)
-		x += 64;
+		x += GetCenterXOffset();
+		y += GetCenterYOffset();
 
 		//add to array
 		RecentFixations.Add(FVector(x, y, points[i].MaxRadius));
@@ -139,20 +162,42 @@ TArray<FVector2D> AActiveSessionView::GetProjectedSaccades()
 		{
 			worldPosition = eyePositions[i]->WorldPosition;
 		}
-		
+
 		FVector2D screenPosition;
-		PlayerController->ProjectWorldLocationToScreen(worldPosition, screenPosition);
+		PlayerController->ProjectWorldLocationToScreen(worldPosition, screenPosition, false);
 
 		//remap from hmd to spectator view
-		float x = Remap(screenPosition.X, 0, 2880, 0, 1280);
-		float y = Remap(screenPosition.Y, 0, 1600, 0, 720);
+		float x = Remap(screenPosition.X, 0, GetHMDWidth(), 0, SpectatorWidth);
+		float y = Remap(screenPosition.Y, 0, GetHMDHeight(), 0, SpectatorHeight);
 
 		//slightly shift the x value (because projection is off center)
-		x += 64;
+		x += GetCenterXOffset();
+		y += GetCenterYOffset();
 
 		//add to array
 		adjustedEyePositions.Add(FVector2D(x, y));
 	}
 
+	//current eye tracking point
+	/*int32 Index = eyePositions.Num() - 1;
+	if (Index > 0)
+	{
+		if (eyePositions[Index]->IsLocal && eyePositions[Index]->Parent != NULL)
+		{
+			worldPosition = eyePositions[Index]->Parent->GetComponentTransform().TransformPosition(eyePositions[Index]->LocalPosition);
+		}
+		else
+		{
+			worldPosition = eyePositions[Index]->WorldPosition;
+		}
+		FVector2D screenPosition;
+		PlayerController->ProjectWorldLocationToScreen(worldPosition, screenPosition);
+
+		//auto provider = FAnalyticsCognitive3D::Get().GetCognitive3DProvider();
+		//provider.Pin()->sensors->RecordSensor("ScreenPosition.X", screenPosition.X);
+		//provider.Pin()->sensors->RecordSensor("ScreenPosition.Y", screenPosition.Y);
+	}
+	*/
+
 	return adjustedEyePositions;
 }

Cognitive3DTest/Plugins/Cognitive3D/Source/Cognitive3D/Private/C3DComponents/FixationRecorder.cpp

@@ -403,6 +403,27 @@ bool UFixationRecorder::AreEyesClosed()
 	return true;
 }
 
+int64 UFixationRecorder::GetEyeCaptureTimestamp()
+{
+	int64 ts = (int64)(FUtil::GetTimestamp() * 1000);
+	return ts;
+}
+#elif defined INCLUDE_OCULUS_PLUGIN
+bool UFixationRecorder::AreEyesClosed()
+{
+	IEyeTracker const* const ET = GEngine ? GEngine->EyeTrackingDevice.Get() : nullptr;
+	if (ET == NULL)
+	{
+		return false;
+	}
+	FEyeTrackerGazeData gazeData;
+	ET->GetEyeTrackerGazeData(gazeData);
+	if (gazeData.ConfidenceValue < 0.5f)
+	{
+		return true;
+	}
+	return false;
+}
 int64 UFixationRecorder::GetEyeCaptureTimestamp()
 {
 	int64 ts = (int64)(FUtil::GetTimestamp() * 1000);
@@ -694,6 +715,41 @@ void UFixationRecorder::TickComponent(float DeltaTime, ELevelTick TickType, FAct
 	{
 		EyeCaptures[index].Discard = true;
 	}
+#elif defined INCLUDE_OCULUS_PLUGIN
+	EyeCaptures[index].EyesClosed = AreEyesClosed();
+	EyeCaptures[index].Time = GetEyeCaptureTimestamp();
+
+	FVector Start = FVector::ZeroVector;
+	FVector WorldDirection = FVector::ZeroVector;
+	FVector End = FVector::ZeroVector;
+	IEyeTracker const* const ET = GEngine ? GEngine->EyeTrackingDevice.Get() : nullptr;
+	if (ET == NULL)
+	{
+		EyeCaptures[index].Discard = true;
+	}
+	else
+	{
+		EEyeTrackerStatus status = ET->GetEyeTrackerStatus();
+		if (status != EEyeTrackerStatus::Tracking)
+		{
+			EyeCaptures[index].Discard = true;
+		}
+		else
+		{
+			FEyeTrackerGazeData gazeData;
+			ET->GetEyeTrackerGazeData(gazeData);
+			if (gazeData.ConfidenceValue < 0.4f)
+			{
+				EyeCaptures[index].Discard = true;
+			}
+			else
+			{
+				WorldDirection = gazeData.GazeDirection;
+				Start = gazeData.GazeOrigin;
+				End = Start + WorldDirection * MaxFixationDistance;
+			}
+		}
+	}
 #elif defined OPENXR_EYETRACKING
 	EyeCaptures[index].EyesClosed = AreEyesClosed();
 	EyeCaptures[index].Time = GetEyeCaptureTimestamp();

Cognitive3DTest/Plugins/Cognitive3D/Source/Cognitive3D/Private/C3DComponents/FixationRecorder.h

@@ -39,6 +39,10 @@
 #if defined HPGLIA_API
 #include "HPGliaClient.h"
 #endif
+#if defined INCLUDE_OCULUS_PLUGIN
+#include "Runtime/EyeTracker/Public/IEyeTracker.h"
+#include "Runtime/EyeTracker/Public/IEyeTrackerModule.h"
+#endif
 #if defined OPENXR_EYETRACKING
 #include "Runtime/EyeTracker/Public/IEyeTracker.h"
 #include "Runtime/EyeTracker/Public/IEyeTrackerModule.h"

Cognitive3DTest/Plugins/Cognitive3D/Source/Cognitive3D/Private/C3DComponents/PlayerTracker.cpp

@@ -110,6 +110,21 @@ FVector UPlayerTracker::GetWorldGazeEnd(FVector start)
 	}
 	End = TempStart + LastDirection * 100000.0f;
 	return End;
+#elif defined INCLUDE_OCULUS_PLUGIN
+
+	FRotator captureRotation = controllers[0]->PlayerCameraManager->GetCameraRotation();
+	FVector End = start + captureRotation.Vector() * 100000.0f;
+	IEyeTracker const* const ET = GEngine ? GEngine->EyeTrackingDevice.Get() : nullptr;
+	if (ET != nullptr)
+	{
+		FEyeTrackerGazeData gazeData;
+		if (ET->GetEyeTrackerGazeData(gazeData))
+		{
+			LastDirection = gazeData.GazeDirection;
+			End = start + LastDirection * 100000.0f;
+		}
+	}
+	return End;
 #elif defined OPENXR_EYETRACKING
 
 	FRotator captureRotation = controllers[0]->PlayerCameraManager->GetCameraRotation();

Cognitive3DTest/Plugins/Cognitive3D/Source/Cognitive3D/Private/C3DComponents/PlayerTracker.h

@@ -39,6 +39,10 @@
 #if defined HPGLIA_API
 #include "HPGliaClient.h"
 #endif
+#if defined INCLUDE_OCULUS_PLUGIN
+#include "Runtime/EyeTracker/Public/IEyeTracker.h"
+#include "Runtime/EyeTracker/Public/IEyeTrackerModule.h"
+#endif
 #if defined OPENXR_EYETRACKING
 #include "Runtime/EyeTracker/Public/IEyeTracker.h"
 #include "Runtime/EyeTracker/Public/IEyeTrackerModule.h"

Cognitive3DTest/Plugins/Cognitive3D/Source/Cognitive3D/Public/ActiveSessionView.h

@@ -18,6 +18,14 @@
 //provides the interface to get data about fixations and eye tracking from fixation recorder to the ASV widget
 //creates the active session view widget and calls 'initialize' to pass a reference to this
 
+UENUM(BlueprintType)
+enum class EActiveSessionViewHMDDisplayType : uint8
+{
+	QuestPro,
+	ViveProEye,
+	Custom
+};
+
 UCLASS()
 class COGNITIVE3D_API AActiveSessionView : public AActor
 {
@@ -27,15 +35,23 @@ class COGNITIVE3D_API AActiveSessionView : public AActor
 	// Sets default values for this actor's properties
 	AActiveSessionView();
 
-	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
-		int32 HMDWidth = 2880;
-	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
-		int32 HMDHeight = 1600;
 	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
 		int32 SpectatorWidth = 1280;
 	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
 		int32 SpectatorHeight = 720;
 
+	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
+		EActiveSessionViewHMDDisplayType HMDDisplayType = EActiveSessionViewHMDDisplayType::QuestPro;
+
+	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
+		int32 CustomHMDWidth = 2880;
+	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
+		int32 CustomHMDHeight = 1600;
+	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
+		int32 CustomCenteringXOffset = 64;
+	UPROPERTY(EditAnywhere, Category = "Cognitive3D Analytics")
+		int32 CustomCenteringYOffset = 0;
+
 	//returns fixations as vector2d screen position. z value is radius
 	UFUNCTION(BlueprintCallable, Category = "Cognitive3D Analytics")
 		TArray<FVector> GetProjectedFixations();
@@ -55,5 +71,80 @@ class COGNITIVE3D_API AActiveSessionView : public AActor
 	void DelaySetupWidget();
 
 private:
-	float Remap(float num, float low1, float high1, float low2, float high2);
+	float Remap(float num, float low1, float high1, float low2, float high2)
+	{
+		return low2 + (num - low1) * (high2 - low2) / (high1 - low1);
+	}
+	int32 GetHMDWidth()
+	{
+		switch (HMDDisplayType)
+		{
+		case EActiveSessionViewHMDDisplayType::QuestPro:
+			return 4320;
+			break;
+		case EActiveSessionViewHMDDisplayType::ViveProEye:
+			return 2880;
+			break;
+		case EActiveSessionViewHMDDisplayType::Custom:
+			return CustomHMDWidth;
+			break;
+		default:
+			break;
+		}
+		return CustomHMDWidth;
+	}
+	int32 GetHMDHeight()
+	{
+		switch (HMDDisplayType)
+		{
+		case EActiveSessionViewHMDDisplayType::QuestPro:
+			return 2224;
+			break;
+		case EActiveSessionViewHMDDisplayType::ViveProEye:
+			return 1600;
+			break;
+		case EActiveSessionViewHMDDisplayType::Custom:
+			return CustomHMDHeight;
+			break;
+		default:
+			break;
+		}
+		return CustomHMDHeight;
+	}
+	int32 GetCenterXOffset()
+	{
+		switch (HMDDisplayType)
+		{
+		case EActiveSessionViewHMDDisplayType::QuestPro:
+			return 192;
+			break;
+		case EActiveSessionViewHMDDisplayType::ViveProEye:
+			return 64;
+			break;
+		case EActiveSessionViewHMDDisplayType::Custom:
+			return CustomCenteringXOffset;
+			break;
+		default:
+			break;
+		}
+		return CustomCenteringXOffset;
+	}
+	int32 GetCenterYOffset()
+	{
+		switch (HMDDisplayType)
+		{
+		case EActiveSessionViewHMDDisplayType::QuestPro:
+			return 0;
+			break;
+		case EActiveSessionViewHMDDisplayType::ViveProEye:
+			return 0;
+			break;
+		case EActiveSessionViewHMDDisplayType::Custom:
+			return CustomCenteringYOffset;
+			break;
+		default:
+			break;
+		}
+		return CustomCenteringYOffset;
+	}
 };