reshade: Add initial reshade support

src/main.cpp

@@ -126,6 +126,9 @@ const struct option *gamescope_options = (struct option[]){
 	{ "hdr-debug-force-output", no_argument, nullptr, 0 },
 	{ "hdr-debug-heatmap", no_argument, nullptr, 0 },
 
+	{ "reshade-effect", required_argument, nullptr, 0 },
+	{ "reshade-technique-idx", required_argument, nullptr, 0 },
+
 	{} // keep last
 };
 
@@ -216,6 +219,10 @@ const char usage[] =
 	"  --hdr-debug-force-output       forces support and output to HDR10 PQ even if the output does not support it (will look very wrong if it doesn't)\n"
 	"  --hdr-debug-heatmap            displays a heatmap-style debug view of HDR luminence across the scene in nits."
 	"\n"
+	"Reshade shader options:\n"
+	"  --reshade-effect               sets the name of a reshade shader to use in either /usr/share/gamescope/reshade/Shaders or ~/.local/share/gamescope/reshade/Shaders\n"
+	"  --reshade-technique-idx        sets technique idx to use from the reshade effect\n"
+	"\n"
 	"Keyboard shortcuts:\n"
 	"  Super + F                      toggle fullscreen\n"
 	"  Super + N                      toggle nearest neighbour filtering\n"

src/meson.build

@@ -110,6 +110,7 @@ src = [
   'log.cpp',
   'ime.cpp',
   'mangoapp.cpp',
+  'reshade_effect_manager.cpp',
 ]
 
 src += spirv_shaders

src/rendervulkan.cpp

@@ -41,6 +41,8 @@
 #include "shaders/ffx_a.h"
 #include "shaders/ffx_fsr1.h"
 
+#include "reshade_effect_manager.hpp"
+
 extern bool g_bWasPartialComposite;
 
 static constexpr mat3x4 g_rgb2yuv_srgb_to_bt601_limited = {{
@@ -258,6 +260,8 @@ bool CVulkanDevice::BInit(VkInstance instance, VkSurfaceKHR surface)
 	std::thread piplelineThread([this](){compileAllPipelines();});
 	piplelineThread.detach();
 
+	g_reshadeManager.init(this);
+
 	return true;
 }
 
@@ -3275,10 +3279,13 @@ bool vulkan_screenshot( const struct FrameInfo_t *frameInfo, std::shared_ptr<CVu
 	return true;
 }
 
+extern std::string g_reshade_effect;
+extern uint32_t g_reshade_technique_idx;
+
 std::unique_ptr<std::thread> defer_wait_thread;
 uint64_t defer_sequence = 0;
 
-bool vulkan_composite( const struct FrameInfo_t *frameInfo, std::shared_ptr<CVulkanTexture> pPipewireTexture, bool partial, bool defer )
+bool vulkan_composite( struct FrameInfo_t *frameInfo, std::shared_ptr<CVulkanTexture> pPipewireTexture, bool partial, bool defer )
 {
 	if ( defer_wait_thread )
 	{
@@ -3289,6 +3296,33 @@ bool vulkan_composite( const struct FrameInfo_t *frameInfo, std::shared_ptr<CVul
 		defer_sequence = 0;
 	}
 
+	if (!g_reshade_effect.empty())
+	{
+		if (frameInfo->layers[0].tex)
+		{
+			ReshadeEffectKey key
+			{
+				.path             = g_reshade_effect,
+				.bufferWidth      = frameInfo->layers[0].tex->width(),
+				.bufferHeight     = frameInfo->layers[0].tex->height(),
+				.bufferColorSpace = frameInfo->layers[0].colorspace,
+				.bufferFormat     = frameInfo->layers[0].tex->format(),
+				.techniqueIdx     = g_reshade_technique_idx,
+			};
+
+			ReshadeEffectPipeline* pipeline = g_reshadeManager.pipeline(key);
+			if (pipeline != nullptr)
+			{
+				uint64_t seq = pipeline->execute(frameInfo->layers[0].tex, &frameInfo->layers[0].tex);
+				g_device.wait(seq);
+			}
+		}
+	}
+	else
+	{
+		g_reshadeManager.clear();
+	}
+
 	auto compositeImage = partial ? g_output.outputImagesPartialOverlay[ g_output.nOutImage ] : g_output.outputImages[ g_output.nOutImage ];
 
 	auto cmdBuffer = g_device.commandBuffer();

src/rendervulkan.hpp

@@ -341,7 +341,7 @@ std::shared_ptr<CVulkanTexture> vulkan_create_texture_from_dmabuf( struct wlr_dm
 std::shared_ptr<CVulkanTexture> vulkan_create_texture_from_bits( uint32_t width, uint32_t height, uint32_t contentWidth, uint32_t contentHeight, uint32_t drmFormat, CVulkanTexture::createFlags texCreateFlags, void *bits );
 std::shared_ptr<CVulkanTexture> vulkan_create_texture_from_wlr_buffer( struct wlr_buffer *buf );
 
-bool vulkan_composite( const struct FrameInfo_t *frameInfo, std::shared_ptr<CVulkanTexture> pScreenshotTexture, bool partial, bool deferred );
+bool vulkan_composite( struct FrameInfo_t *frameInfo, std::shared_ptr<CVulkanTexture> pScreenshotTexture, bool partial, bool deferred );
 std::shared_ptr<CVulkanTexture> vulkan_get_last_output_image( bool partial, bool defer );
 std::shared_ptr<CVulkanTexture> vulkan_acquire_screenshot_texture(uint32_t width, uint32_t height, bool exportable, uint32_t drmFormat, EStreamColorspace colorspace = k_EStreamColorspace_Unknown);
 

src/reshade_effect_manager.hpp

@@ -0,0 +1,96 @@
+#pragma once
+
+#include "rendervulkan.hpp"
+#include <optional>
+
+namespace reshadefx
+{
+    class module;
+}
+
+class ReshadeUniform;
+
+struct ReshadeCombinedImageSampler
+{
+    VkSampler sampler;
+    std::shared_ptr<CVulkanTexture> texture;
+};
+
+struct ReshadeEffectKey
+{
+	std::string path;
+
+	uint32_t bufferWidth;
+	uint32_t bufferHeight;
+	GamescopeAppTextureColorspace bufferColorSpace;
+    VkFormat bufferFormat;
+
+	uint32_t techniqueIdx;
+
+    bool operator==(const ReshadeEffectKey& other) const = default;
+    bool operator!=(const ReshadeEffectKey& other) const = default;
+};
+
+enum ReshadeDescriptorSets
+{
+    GAMESCOPE_RESHADE_DESCRIPTOR_SET_UBO = 0,
+    GAMESCOPE_RESHADE_DESCRIPTOR_SET_SAMPLED_IMAGES,
+    GAMESCOPE_RESHADE_DESCRIPTOR_SET_STORAGE_IMAGES,
+
+    GAMESCOPE_RESHADE_DESCRIPTOR_SET_COUNT,
+};
+
+class ReshadeEffectPipeline
+{
+public:
+    ReshadeEffectPipeline();
+    ~ReshadeEffectPipeline();
+
+    bool init(CVulkanDevice *device, const ReshadeEffectKey &key);
+    void update();
+    uint64_t execute(std::shared_ptr<CVulkanTexture> inImage, std::shared_ptr<CVulkanTexture> *outImage);
+
+    const ReshadeEffectKey& key() const { return m_key; }
+    reshadefx::module *module() { return m_module.get(); }
+
+    std::shared_ptr<CVulkanTexture> findTexture(std::string_view name);
+
+private:
+    ReshadeEffectKey m_key;
+    CVulkanDevice *m_device;
+
+	std::unique_ptr<reshadefx::module> m_module;
+    std::vector<VkPipeline> m_pipelines;
+    std::vector<std::shared_ptr<CVulkanTexture>> m_textures;
+    std::shared_ptr<CVulkanTexture> m_rt;
+    std::vector<ReshadeCombinedImageSampler> m_samplers;
+    std::vector<std::shared_ptr<ReshadeUniform>> m_uniforms;
+
+    std::optional<CVulkanCmdBuffer> m_cmdBuffer = std::nullopt;
+    VkBuffer m_buffer = VK_NULL_HANDLE;
+    VkDeviceMemory m_bufferMemory = VK_NULL_HANDLE;
+    void* m_mappedPtr = nullptr;
+
+    VkPipelineLayout m_pipelineLayout = VK_NULL_HANDLE;
+    VkDescriptorPool m_descriptorPool = VK_NULL_HANDLE;
+
+    VkDescriptorSetLayout m_descriptorSetLayouts[GAMESCOPE_RESHADE_DESCRIPTOR_SET_COUNT] = {};
+    VkDescriptorSet m_descriptorSets[GAMESCOPE_RESHADE_DESCRIPTOR_SET_COUNT] = {};
+};
+
+class ReshadeEffectManager
+{
+public:
+    ReshadeEffectManager();
+
+    void init(CVulkanDevice *device);
+    void clear();
+    ReshadeEffectPipeline* pipeline(const ReshadeEffectKey &key);
+
+private:
+    ReshadeEffectKey m_lastKey{};
+    std::unique_ptr<ReshadeEffectPipeline> m_lastPipeline;
+    CVulkanDevice *m_device;
+};
+
+extern ReshadeEffectManager g_reshadeManager;

src/steamcompmgr.cpp

@@ -133,6 +133,9 @@ extern std::atomic<uint64_t> g_lastVblank;
 std::string clipboard;
 std::string primarySelection;
 
+std::string g_reshade_effect{};
+uint32_t g_reshade_technique_idx = 0;
+
 uint64_t timespec_to_nanos(struct timespec& spec)
 {
 	return spec.tv_sec * 1'000'000'000ul + spec.tv_nsec;
@@ -5703,6 +5706,16 @@ handle_property_notify(xwayland_ctx_t *ctx, XPropertyEvent *ev)
 			focusDirty = true;
 		}
 	}
+	if (ev->atom == ctx->atoms.gamescopeReshadeTechniqueIdx)
+	{
+		uint32_t technique_idx = get_prop(ctx, ctx->root, ctx->atoms.gamescopeReshadeTechniqueIdx, 0);
+		g_reshade_technique_idx = technique_idx;
+	}
+	if (ev->atom == ctx->atoms.gamescopeReshadeEffect)
+	{
+		std::string path = get_string_prop( ctx, ctx->root, ctx->atoms.gamescopeReshadeEffect );
+		g_reshade_effect = path;
+	}
 	if (ev->atom == ctx->atoms.wineHwndStyle)
 	{
 		steamcompmgr_win_t * w = find_win(ctx, ev->window);
@@ -6863,6 +6876,9 @@ void init_xwayland_ctx(uint32_t serverId, gamescope_xwayland_server_t *xwayland_
 	ctx->atoms.gamescopeCreateXWaylandServerFeedback = XInternAtom( ctx->dpy, "GAMESCOPE_CREATE_XWAYLAND_SERVER_FEEDBACK", false );
 	ctx->atoms.gamescopeDestroyXWaylandServer = XInternAtom( ctx->dpy, "GAMESCOPE_DESTROY_XWAYLAND_SERVER", false );
 
+	ctx->atoms.gamescopeReshadeEffect = XInternAtom( ctx->dpy, "GAMESCOPE_RESHADE_EFFECT", false );
+	ctx->atoms.gamescopeReshadeTechniqueIdx = XInternAtom( ctx->dpy, "GAMESCOPE_RESHADE_TECHNIQUE_IDX", false );
+
 	ctx->atoms.wineHwndStyle = XInternAtom( ctx->dpy, "_WINE_HWND_STYLE", false );
 	ctx->atoms.wineHwndStyleEx = XInternAtom( ctx->dpy, "_WINE_HWND_EXSTYLE", false );
 
@@ -7154,6 +7170,10 @@ steamcompmgr_main(int argc, char **argv)
 					g_flHDRItmTargetNits = atof(optarg);
 				} else if (strcmp(opt_name, "framerate-limit") == 0) {
 					g_nSteamCompMgrTargetFPS = atoi(optarg);
+				} else if (strcmp(opt_name, "reshade-effect") == 0) {
+					g_reshade_effect = optarg;
+				} else if (strcmp(opt_name, "reshade-technique-idx") == 0) {
+					g_reshade_technique_idx = atoi(optarg);
 				}
 				break;
 			case '?':

src/xwayland_ctx.hpp

@@ -207,6 +207,9 @@ struct xwayland_ctx_t
 		Atom gamescopeCreateXWaylandServerFeedback;
 		Atom gamescopeDestroyXWaylandServer;
 
+		Atom gamescopeReshadeEffect;
+		Atom gamescopeReshadeTechniqueIdx;
+
 		Atom wineHwndStyle;
 		Atom wineHwndStyleEx;