attempt to fix issue ValveSoftware#1218

sharkautarch · Mar 27, 2024 · 4a05d0d · 4a05d0d
1 parent 9563271
commit 4a05d0d
Showing 1 changed file with 19 additions and 3 deletions.
diff --git a/src/wayland_backend.cpp b/src/wayland_backend.cpp
@@ -892,8 +892,24 @@ namespace gamescope
 
     void CWaylandBackend::GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const
     {
-        *pPrimaryPlaneFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
-        *pOverlayPlaneFormat = VK_FORMAT_B8G8R8A8_UNORM;
+        VkFormat u8BitFormat = VK_FORMAT_UNDEFINED;
+        if ( m_FormatModifiers.contains( DRM_FORMAT_ARGB8888 ) )
+            u8BitFormat = VK_FORMAT_B8G8R8A8_UNORM;
+        else if ( m_FormatModifiers.contains( DRM_FORMAT_ABGR8888 ) )
+            u8BitFormat = VK_FORMAT_R8G8B8A8_UNORM;
+
+        VkFormat u10BitFormat = VK_FORMAT_UNDEFINED;
+        if ( m_FormatModifiers.contains( DRM_FORMAT_ABGR2101010 ) )
+            u10BitFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
+        else if ( m_FormatModifiers.contains( DRM_FORMAT_ARGB2101010 ) )
+            u10BitFormat = VK_FORMAT_A2R10G10B10_UNORM_PACK32;
+
+		if ( u10BitFormat != VK_FORMAT_UNDEFINED && u8BitFormat == VK_FORMAT_UNDEFINED)
+			u8BitFormat = u10BitFormat; 
+        assert( u8BitFormat != VK_FORMAT_UNDEFINED );
+
+        *pPrimaryPlaneFormat = u10BitFormat != VK_FORMAT_UNDEFINED ? u10BitFormat : u8BitFormat;
+        *pOverlayPlaneFormat = u8BitFormat;
     }
 
     bool CWaylandBackend::ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const
@@ -1071,7 +1087,7 @@ namespace gamescope
     {
         auto iter = m_FormatModifiers.find( uDrmFormat );
         if ( iter == m_FormatModifiers.end() )
-            return std::span<const uint64_t>{};
+            return std::span<const uint64_t>{ {g_device.supportsModifiers() ? 0 : DRM_FORMAT_MOD_INVALID} };
 
         return std::span<const uint64_t>{ iter->second.begin(), iter->second.end() };
     }