drm, rendervulkan: Pick a 10-bit format for composite buffer

1. Many game engines automatically render to 10-bit formats such as UE4 which means
that when we have to composite, we can keep the same HW dithering that we would get if
we just scanned them out directly.

2. When compositing HDR content as a fallback when we undock, it avoids introducing
a bunch of horrible banding when going to G2.2 curve.
It ensures that we can dither that.

src/drm.cpp

@@ -41,7 +41,6 @@ extern "C" {
 struct drm_t g_DRM = {};
 
 uint32_t g_nDRMFormat = DRM_FORMAT_INVALID;
-uint32_t g_nDRMFormatHDR = DRM_FORMAT_INVALID;
 bool g_bRotated = false;
 bool g_bUseLayers = true;
 bool g_bDebugLayers = false;
@@ -1194,17 +1193,24 @@ bool init_drm(struct drm_t *drm, int width, int height, int refresh, bool wants_
 		return false;
 	}
 
-	g_nDRMFormat = pick_plane_format(&drm->primary_formats, DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888);
+	// Pick a 10-bit format at first for our composition buffer, for a couple of reasons:
+	//
+	// 1. Many game engines automatically render to 10-bit formats such as UE4 which means
+	// that when we have to composite, we can keep the same HW dithering that we would get if
+	// we just scanned them out directly.
+	//
+	// 2. When compositing HDR content as a fallback when we undock, it avoids introducing
+	// a bunch of horrible banding when going to G2.2 curve.
+	// It ensures that we can dither that.
+	g_nDRMFormat = pick_plane_format(&drm->primary_formats, DRM_FORMAT_XRGB2101010, DRM_FORMAT_ARGB2101010);
 	if ( g_nDRMFormat == DRM_FORMAT_INVALID ) {
-		drm_log.errorf("Primary plane doesn't support XRGB8888 nor ARGB8888");
+		g_nDRMFormat = pick_plane_format(&drm->primary_formats, DRM_FORMAT_XBGR2101010, DRM_FORMAT_ABGR2101010);
-		return false;
+		if ( g_nDRMFormat == DRM_FORMAT_INVALID ) {
-	}
+			g_nDRMFormat = pick_plane_format(&drm->primary_formats, DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888);
-
+			if ( g_nDRMFormat == DRM_FORMAT_INVALID ) {
-	g_nDRMFormatHDR = pick_plane_format(&drm->primary_formats, DRM_FORMAT_XRGB2101010, DRM_FORMAT_ARGB2101010);
+				drm_log.errorf("Primary plane doesn't support any formats >= 8888");
-	if ( g_nDRMFormatHDR == DRM_FORMAT_INVALID ) {
+				return false;
-		g_nDRMFormatHDR = pick_plane_format(&drm->primary_formats, DRM_FORMAT_XBGR2101010, DRM_FORMAT_ABGR2101010);
+			}
-		if ( g_nDRMFormatHDR == DRM_FORMAT_INVALID ) {
-			drm_log.errorf("Primary plane doesn't support any 10 bit formats. No HDR.");
 		}
 	}
 
@@ -2770,9 +2776,6 @@ bool drm_get_vrr_capable(struct drm_t *drm)
 
 bool drm_supports_st2084(struct drm_t *drm)
 {
-	if ( g_nDRMFormatHDR == DRM_FORMAT_INVALID )
-		return false;
-
 	if ( drm->connector )
 		return drm->connector->metadata.supportsST2084;
 

src/drm.hpp

@@ -270,7 +270,6 @@ struct drm_t {
 extern struct drm_t g_DRM;
 
 extern uint32_t g_nDRMFormat;
-extern uint32_t g_nDRMFormatHDR;
 
 extern bool g_bUseLayers;
 extern bool g_bRotated;

src/rendervulkan.cpp

@@ -106,7 +106,6 @@ struct VulkanOutput_t
 	std::vector<std::shared_ptr<CVulkanTexture>> outputImages;
 
 	VkFormat outputFormat = VK_FORMAT_UNDEFINED;
-	VkFormat outputFormatHDR = VK_FORMAT_UNDEFINED;
 
 	std::array<std::shared_ptr<CVulkanTexture>, 8> pScreenshotImages;
 
@@ -2983,7 +2982,7 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput )
 	pOutput->outputImages[0] = nullptr;
 	pOutput->outputImages[1] = nullptr;
 
-	VkFormat format = g_bOutputHDREnabled ? pOutput->outputFormatHDR : pOutput->outputFormat;
+	VkFormat format = pOutput->outputFormat;
 
 	pOutput->outputImages[0] = std::make_shared<CVulkanTexture>();
 	bool bSuccess = pOutput->outputImages[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(format), outputImageflags );
@@ -3028,7 +3027,7 @@ bool vulkan_make_output( VkSurfaceKHR surface )
 	
 	if ( BIsVRSession() )
 	{
-		pOutput->outputFormat = VK_FORMAT_B8G8R8A8_UNORM;
+		pOutput->outputFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
 		vulkan_make_output_images( pOutput );
 	}
 	else if ( BIsNested() == true )
@@ -3088,7 +3087,6 @@ bool vulkan_make_output( VkSurfaceKHR surface )
 	else
 	{
 		pOutput->outputFormat = DRMFormatToVulkan( g_nDRMFormat, false );
-		pOutput->outputFormatHDR = DRMFormatToVulkan( g_nDRMFormatHDR, false );
 		
 		if ( pOutput->outputFormat == VK_FORMAT_UNDEFINED )
 		{

src/steamcompmgr.cpp

@@ -6881,7 +6881,6 @@ steamcompmgr_main(int argc, char **argv)
 						drm_set_hdr_state(&g_DRM, g_bOutputHDREnabled);
 					}
 				}
-				vulkan_remake_output_images();
 			}