drm: Respect app's HDR metadata

src/drm.cpp

@@ -1845,7 +1845,16 @@ int drm_prepare( struct drm_t *drm, bool async, const struct FrameInfo_t *frameI
 		if (ret < 0)
 			return ret;
 
-		drm->connector->pending.hdr_output_metadata = g_bOutputHDREnabled ? drm->connector->metadata.hdr10_metadata_blob : 0;
+		uint32_t hdr_output_metadata_blob = 0;
+		if ( g_bOutputHDREnabled ) {
+			hdr_output_metadata_blob = drm->connector->metadata.hdr10_metadata_blob;
+
+			auto feedback = steamcompmgr_get_base_layer_swapchain_feedback();
+			if (feedback && feedback->hdr_metadata_blob)
+				hdr_output_metadata_blob = feedback->hdr_metadata_blob;
+		}
+
+		drm->connector->pending.hdr_output_metadata = g_bOutputHDREnabled ? hdr_output_metadata_blob : 0;
 		ret = add_connector_property(drm->req, drm->connector, "HDR_OUTPUT_METADATA", drm->connector->pending.hdr_output_metadata);
 		if (ret < 0)
 			return ret;
@@ -2559,3 +2568,20 @@ std::pair<uint32_t, uint32_t> drm_get_connector_identifier(struct drm_t *drm)
 
 	return std::make_pair(drm->connector->connector->connector_type, drm->connector->connector->connector_type_id);
 }
+
+uint32_t drm_create_hdr_metadata_blob(struct drm_t *drm, hdr_output_metadata *metadata)
+{
+	uint32_t blob = 0;
+	int ret = drmModeCreatePropertyBlob(drm->fd, metadata, sizeof(*metadata), &blob);
+
+	if (ret != 0) {
+		drm_log.errorf("Failed to create blob for HDR_OUTPUT_METADATA. (%s) Falling back to null blob.", strerror(-ret));
+		blob = 0;
+	}
+
+	return blob;
+}
+void drm_destroy_hdr_metadata_blob(struct drm_t *drm, uint32_t blob)
+{
+	drmModeDestroyPropertyBlob(drm->fd, blob);
+}

src/drm.hpp

@@ -6,6 +6,7 @@
 #include <xf86drmMode.h>
 #include <assert.h>
 #include <drm_fourcc.h>
+#include <drm_mode.h>
 #include "color_helpers.h"
 
 // Josh: Okay whatever, this header isn't
@@ -270,6 +271,8 @@ bool drm_get_vrr_capable(struct drm_t *drm);
 bool drm_supports_st2084(struct drm_t *drm);
 void drm_set_vrr_enabled(struct drm_t *drm, bool enabled);
 bool drm_get_vrr_in_use(struct drm_t *drm);
+uint32_t drm_create_hdr_metadata_blob(struct drm_t *drm, hdr_output_metadata *metadata);
+void drm_destroy_hdr_metadata_blob(struct drm_t *drm, uint32_t blob);
 
 const char *drm_get_connector_name(struct drm_t *drm);
 const char *drm_get_device_name(struct drm_t *drm);

src/steamcompmgr.cpp

@@ -1478,6 +1478,17 @@ namespace PaintWindowFlag
 }
 using PaintWindowFlags = uint32_t;
 
+wlserver_vk_swapchain_feedback* steamcompmgr_get_base_layer_swapchain_feedback()
+{
+	if ( !g_HeldCommits[ HELD_COMMIT_BASE ] )
+		return nullptr;
+
+	if ( !g_HeldCommits[ HELD_COMMIT_BASE ]->feedback )
+		return nullptr;
+
+	return &(*g_HeldCommits[ HELD_COMMIT_BASE ]->feedback);
+}
+
 static void
 paint_window(win *w, win *scaleW, struct FrameInfo_t *frameInfo,
 			  MouseCursor *cursor, PaintWindowFlags flags = 0, float flOpacityScale = 1.0f, win *fit = nullptr )

src/steamcompmgr.hpp

@@ -16,6 +16,7 @@ void sleep_until_nanos(uint64_t nanos);
 void steamcompmgr_main(int argc, char **argv);
 
 #include "rendervulkan.hpp"
+#include "wlserver.hpp"
 
 #include <mutex>
 #include <vector>
@@ -120,6 +121,7 @@ void force_repaint( void );
 extern void mangoapp_update( uint64_t visible_frametime, uint64_t app_frametime_ns, uint64_t latency_ns );
 gamescope_xwayland_server_t *steamcompmgr_get_focused_server();
 struct wlr_surface *steamcompmgr_get_server_input_surface( size_t idx );
+wlserver_vk_swapchain_feedback* steamcompmgr_get_base_layer_swapchain_feedback();
 
 struct wlserver_x11_surface_info *lookup_x11_surface_info_from_xid( gamescope_xwayland_server_t *xwayland_server, uint32_t xid );
 

src/wlserver.cpp

@@ -546,6 +546,7 @@ static void gamescope_xwayland_handle_swapchain_feedback( struct wl_client *clie
 			.vk_pre_transform = VkSurfaceTransformFlagBitsKHR(vk_pre_transform),
 			.vk_present_mode = VkPresentModeKHR(vk_present_mode),
 			.vk_clipped = VkBool32(vk_clipped),
+			.hdr_metadata_blob = 0,
 		});
 	}
 }
@@ -567,9 +568,43 @@ static void gamescope_xwayland_handle_set_hdr_metadata( struct wl_client *client
 {
 	struct wlr_surface *surface = wlr_surface_from_resource( surface_resource );
 	wlserver_wl_surface_info *wl_info = get_wl_surface_info( surface );
+	if ( BIsNested() )
+	{
+		wl_log.infof("Ignoring HDR metadata when nested.");
+		return;
+	}
+
 	if ( wl_info )
 	{
-		fprintf(stderr, "GOT HDR METADATA FOR SURFACE!\n");
+		if ( !wl_info->swapchain_feedback ) {
+			wl_log.errorf("set_hdr_metadata with no swapchain_feedback.");
+			return;
+		}
+
+		drm_destroy_hdr_metadata_blob( &g_DRM, wl_info->swapchain_feedback->hdr_metadata_blob );
+		wl_info->swapchain_feedback->hdr_metadata_blob = 0;
+
+		hdr_output_metadata metadata = {};
+		metadata.metadata_type = 0;
+
+		hdr_metadata_infoframe& infoframe = metadata.hdmi_metadata_type1;
+		infoframe.eotf = HDMI_EOTF_ST2084;
+		infoframe.metadata_type = 0;
+		infoframe.display_primaries[0].x = display_primary_red_x;
+		infoframe.display_primaries[0].y = display_primary_red_y;
+		infoframe.display_primaries[1].x = display_primary_green_x;
+		infoframe.display_primaries[1].y = display_primary_green_y;
+		infoframe.display_primaries[2].x = display_primary_blue_x;
+		infoframe.display_primaries[2].y = display_primary_blue_y;
+		infoframe.white_point.x = white_point_x;
+		infoframe.white_point.y = white_point_y;
+		infoframe.max_display_mastering_luminance = max_display_mastering_luminance;
+		infoframe.min_display_mastering_luminance = min_display_mastering_luminance;
+		infoframe.max_cll = max_cll;
+		infoframe.max_fall = max_fall;
+
+		wl_info->swapchain_feedback->hdr_metadata_blob =
+			drm_create_hdr_metadata_blob( &g_DRM, &metadata );
 	}
 }
 

src/wlserver.hpp

@@ -12,6 +12,8 @@
 #include <optional>
 #include <vulkan/vulkan_core.h>
 
+#include "drm.hpp"
+
 #define WLSERVER_BUTTON_COUNT 4
 
 struct _XDisplay;
@@ -19,6 +21,11 @@ struct xwayland_ctx_t;
 
 struct wlserver_vk_swapchain_feedback
 {
+	~wlserver_vk_swapchain_feedback()
+	{
+		drm_destroy_hdr_metadata_blob( &g_DRM, hdr_metadata_blob );
+	}
+
 	uint32_t image_count;
 	VkFormat vk_format;
 	VkColorSpaceKHR vk_colorspace;
@@ -26,6 +33,7 @@ struct wlserver_vk_swapchain_feedback
 	VkSurfaceTransformFlagBitsKHR vk_pre_transform;
 	VkPresentModeKHR vk_present_mode;
 	VkBool32 vk_clipped;
+	uint32_t hdr_metadata_blob;
 };
 
 struct ResListEntry_t {