drm: Implement COLOR_ENCODING + COLOR_RANGE and hook up to env var for now

There is no way to tel a NV12 buffer in Wayland's color space. Why?
What a glaring omission.

Even then, the color management protocol that has supposedly been in
development for 2 years is missing basic features like setting the
color range. Great!

For now we are just setting an env var both us and Remote Play Together
are going to listen to and call it a day until the situation is
properly resolved.

src/drm.cpp

@@ -1167,6 +1167,47 @@ drm_prepare_basic( struct drm_t *drm, const struct FrameInfo_t *frameInfo )
 	return ret;
 }
 
+// Only used for NV12 buffers
+static drm_color_encoding drm_get_color_encoding(EStreamColorspace colorspace)
+{
+	switch (colorspace)
+	{
+		default:
+		case k_EStreamColorspace_Unknown:
+			return DRM_COLOR_YCBCR_BT709;
+
+		case k_EStreamColorspace_BT601:
+			return DRM_COLOR_YCBCR_BT601;
+		case k_EStreamColorspace_BT601_Full:
+			return DRM_COLOR_YCBCR_BT601;
+
+		case k_EStreamColorspace_BT709:
+			return DRM_COLOR_YCBCR_BT709;
+		case k_EStreamColorspace_BT709_Full:
+			return DRM_COLOR_YCBCR_BT709;
+	}
+}
+
+static drm_color_range drm_get_color_range(EStreamColorspace colorspace)
+{
+	switch (colorspace)
+	{
+		default:
+		case k_EStreamColorspace_Unknown:
+			return DRM_COLOR_YCBCR_FULL_RANGE;
+
+		case k_EStreamColorspace_BT601:
+			return DRM_COLOR_YCBCR_LIMITED_RANGE;
+		case k_EStreamColorspace_BT601_Full:
+			return DRM_COLOR_YCBCR_FULL_RANGE;
+
+		case k_EStreamColorspace_BT709:
+			return DRM_COLOR_YCBCR_LIMITED_RANGE;
+		case k_EStreamColorspace_BT709_Full:
+			return DRM_COLOR_YCBCR_FULL_RANGE;
+	}
+}
+
 static int
 drm_prepare_liftoff( struct drm_t *drm, const struct FrameInfo_t *frameInfo )
 {
@@ -1217,6 +1258,9 @@ drm_prepare_liftoff( struct drm_t *drm, const struct FrameInfo_t *frameInfo )
 
 			liftoff_layer_set_property( drm->lo_layers[ i ], "CRTC_W", crtcW);
 			liftoff_layer_set_property( drm->lo_layers[ i ], "CRTC_H", crtcH);
+
+			liftoff_layer_set_property( drm->lo_layers[ i ], "COLOR_ENCODING", drm_get_color_encoding( g_ForcedNV12ColorSpace ) );
+			liftoff_layer_set_property( drm->lo_layers[ i ], "COLOR_RANGE", drm_get_color_range( g_ForcedNV12ColorSpace ) );
 		}
 		else
 		{

src/drm.hpp

@@ -6,6 +6,23 @@
 #include <xf86drmMode.h>
 #include <assert.h>
 #include <drm_fourcc.h>
+
+// Josh: Okay whatever, this header isn't
+// available for whatever stupid reason. :v
+//#include <drm_color_mgmt.h>
+enum drm_color_encoding {
+	DRM_COLOR_YCBCR_BT601,
+	DRM_COLOR_YCBCR_BT709,
+	DRM_COLOR_YCBCR_BT2020,
+	DRM_COLOR_ENCODING_MAX,
+};
+
+enum drm_color_range {
+	DRM_COLOR_YCBCR_LIMITED_RANGE,
+	DRM_COLOR_YCBCR_FULL_RANGE,
+	DRM_COLOR_RANGE_MAX,
+};
+
 #include <wayland-server-core.h>
 
 extern "C" {

src/main.cpp

@@ -27,6 +27,8 @@
 #include "pipewire.hpp"
 #endif
 
+EStreamColorspace g_ForcedNV12ColorSpace = k_EStreamColorspace_Unknown;
+
 const char *gamescope_optstring = nullptr;
 
 const struct option *gamescope_options = (struct option[]){
@@ -269,6 +271,23 @@ static void raise_fd_limit( void )
 	g_fdLimitRaised = true;
 }
 
+static EStreamColorspace parse_colorspace_string( const char *pszStr )
+{
+	if ( !pszStr || !*pszStr )
+		return k_EStreamColorspace_Unknown;
+
+	if ( !strcmp( pszStr, "k_EStreamColorspace_BT601" ) )
+		return k_EStreamColorspace_BT601;
+	else if ( !strcmp( pszStr, "k_EStreamColorspace_BT601_Full" ) )
+		return k_EStreamColorspace_BT601_Full;
+	else if ( !strcmp( pszStr, "k_EStreamColorspace_BT709" ) )
+		return k_EStreamColorspace_BT709;
+	else if ( !strcmp( pszStr, "k_EStreamColorspace_BT709_Full" ) )
+		return k_EStreamColorspace_BT709_Full;
+	else
+	 	return k_EStreamColorspace_Unknown;
+}
+
 int g_nPreferredOutputWidth = 0;
 int g_nPreferredOutputHeight = 0;
 
@@ -434,6 +453,8 @@ int main(int argc, char **argv)
 		}
 	}
 
+	g_ForcedNV12ColorSpace = parse_colorspace_string( getenv( "GAMESCOPE_NV12_COLORSPACE" ) );
+
 	if ( !vulkan_init() )
 	{
 		fprintf( stderr, "Failed to initialize Vulkan\n" );

src/rendervulkan.cpp

@@ -9,6 +9,7 @@
 #include <array>
 #include <bitset>
 #include <thread>
+#include <vulkan/vulkan_core.h>
 
 // Used to remove the config struct alignment specified by the NIS header
 #define NIS_ALIGNED(x)
@@ -835,6 +836,42 @@ bool CVulkanDevice::createDevice()
 	return true;
 }
 
+static VkSamplerYcbcrModelConversion colorspaceToYCBCRModel( EStreamColorspace colorspace )
+{
+	switch (colorspace)
+	{
+		default:
+		case k_EStreamColorspace_Unknown:
+			return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709;
+
+		case k_EStreamColorspace_BT601:
+		case k_EStreamColorspace_BT601_Full:
+			return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_601;
+
+		case k_EStreamColorspace_BT709:
+		case k_EStreamColorspace_BT709_Full:
+			return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709;
+	}
+}
+
+static VkSamplerYcbcrRange colorspaceToYCBCRRange( EStreamColorspace colorspace )
+{
+	switch (colorspace)
+	{
+		default:
+		case k_EStreamColorspace_Unknown:
+			return VK_SAMPLER_YCBCR_RANGE_ITU_FULL;
+
+		case k_EStreamColorspace_BT709:
+		case k_EStreamColorspace_BT601:
+			return VK_SAMPLER_YCBCR_RANGE_ITU_NARROW;
+
+		case k_EStreamColorspace_BT601_Full:
+		case k_EStreamColorspace_BT709_Full:
+			return VK_SAMPLER_YCBCR_RANGE_ITU_FULL;
+	}
+}
+
 bool CVulkanDevice::createLayouts()
 {
 	VkFormatProperties nv12Properties;
@@ -845,8 +882,8 @@ bool CVulkanDevice::createLayouts()
 	{
 		.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_CREATE_INFO,
 		.format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM,
-		.ycbcrModel = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709,
-		.ycbcrRange = VK_SAMPLER_YCBCR_RANGE_ITU_FULL,
+		.ycbcrModel = colorspaceToYCBCRModel( g_ForcedNV12ColorSpace ),
+		.ycbcrRange = colorspaceToYCBCRRange( g_ForcedNV12ColorSpace ),
 		.xChromaOffset = cosited ? VK_CHROMA_LOCATION_COSITED_EVEN : VK_CHROMA_LOCATION_MIDPOINT,
 		.yChromaOffset = cosited ? VK_CHROMA_LOCATION_COSITED_EVEN : VK_CHROMA_LOCATION_MIDPOINT,
 		.chromaFilter = VK_FILTER_LINEAR,

src/rendervulkan.hpp

@@ -34,6 +34,14 @@ enum BlurMode {
     BLUR_MODE_ALWAYS = 2,
 };
 
+enum EStreamColorspace : int
+{
+	k_EStreamColorspace_Unknown = 0,
+	k_EStreamColorspace_BT601 = 1,
+	k_EStreamColorspace_BT601_Full = 2,
+	k_EStreamColorspace_BT709 = 3,
+	k_EStreamColorspace_BT709_Full = 4
+};
 
 #include "drm.hpp"
 

src/steamcompmgr.hpp

@@ -35,6 +35,8 @@ static const uint32_t g_zposExternalOverlay = 2;
 static const uint32_t g_zposOverlay = 3;
 static const uint32_t g_zposCursor = 4;
 
+extern EStreamColorspace g_ForcedNV12ColorSpace;
+
 class MouseCursor
 {
 public: