steamcompmgr: Add support for growing/destroying xwaylands on the fly

src/steamcompmgr.cpp

@@ -213,6 +213,8 @@ struct commit_t
 	std::optional<wlserver_vk_swapchain_feedback> feedback = std::nullopt;
 };
 
+static std::vector<pollfd> pollfds;
+
 #define MWM_HINTS_FUNCTIONS   1
 #define MWM_HINTS_DECORATIONS 2
 #define MWM_HINTS_INPUT_MODE  4
@@ -4745,6 +4747,94 @@ handle_property_notify(xwayland_ctx_t *ctx, XPropertyEvent *ev)
 				hasRepaint = true;
 		}
 	}
+	if (ev->atom == ctx->atoms.gamescopeCreateXWaylandServer)
+	{
+		uint32_t identifier = get_prop(ctx, ctx->root, ctx->atoms.gamescopeCreateXWaylandServer, 0);
+		if (identifier)
+		{
+			wlserver_lock();
+			uint32_t server_id = (uint32_t)wlserver_make_new_xwayland_server();
+			assert(server_id != ~0u);
+			gamescope_xwayland_server_t *server = wlserver_get_xwayland_server(server_id);
+			init_xwayland_ctx(server_id, server);
+			char propertyString[256];
+			snprintf(propertyString, sizeof(propertyString), "%u %s", identifier, server->get_nested_display_name());
+			XTextProperty text_property =
+			{
+				.value = (unsigned char *)propertyString,
+				.encoding = ctx->atoms.utf8StringAtom,
+				.format = 8,
+				.nitems = strlen(propertyString),
+			};
+			pollfds.push_back(pollfd {
+				.fd = XConnectionNumber( server->ctx->dpy ),
+				.events = POLLIN,
+			});
+			XSetTextProperty( ctx->dpy, ctx->root, &text_property, ctx->atoms.gamescopeCreateXWaylandServerFeedback );
+			wlserver_unlock();
+		}
+	}
+	if (ev->atom == ctx->atoms.gamescopeDestroyXWaylandServer)
+	{
+		uint32_t server_id = get_prop(ctx, ctx->root, ctx->atoms.gamescopeDestroyXWaylandServer, 0);
+
+		gamescope_xwayland_server_t *server = wlserver_get_xwayland_server(server_id);
+		if (server)
+		{
+			if (global_focus.focusWindow &&
+				global_focus.focusWindow->type == steamcompmgr_win_type_t::XWAYLAND &&
+				global_focus.focusWindow->xwayland().ctx == server->ctx.get())
+				global_focus.focusWindow = nullptr;
+
+			if (global_focus.inputFocusWindow &&
+				global_focus.inputFocusWindow->type == steamcompmgr_win_type_t::XWAYLAND &&
+				global_focus.inputFocusWindow->xwayland().ctx == server->ctx.get())
+				global_focus.inputFocusWindow = nullptr;
+
+			if (global_focus.overlayWindow &&
+				global_focus.overlayWindow->type == steamcompmgr_win_type_t::XWAYLAND &&
+				global_focus.overlayWindow->xwayland().ctx == server->ctx.get())
+				global_focus.overlayWindow = nullptr;
+
+			if (global_focus.externalOverlayWindow &&
+				global_focus.externalOverlayWindow->type == steamcompmgr_win_type_t::XWAYLAND &&
+				global_focus.externalOverlayWindow->xwayland().ctx == server->ctx.get())
+				global_focus.externalOverlayWindow = nullptr;
+
+			if (global_focus.notificationWindow &&
+				global_focus.notificationWindow->type == steamcompmgr_win_type_t::XWAYLAND &&
+				global_focus.notificationWindow->xwayland().ctx == server->ctx.get())
+				global_focus.notificationWindow = nullptr;
+
+			if (global_focus.overrideWindow &&
+				global_focus.overrideWindow->type == steamcompmgr_win_type_t::XWAYLAND &&
+				global_focus.overrideWindow->xwayland().ctx == server->ctx.get())
+				global_focus.overrideWindow = nullptr;
+
+			if (global_focus.keyboardFocusWindow &&
+				global_focus.keyboardFocusWindow->type == steamcompmgr_win_type_t::XWAYLAND &&
+				global_focus.keyboardFocusWindow->xwayland().ctx == server->ctx.get())
+				global_focus.keyboardFocusWindow = nullptr;
+
+			if (global_focus.fadeWindow &&
+				global_focus.fadeWindow->type == steamcompmgr_win_type_t::XWAYLAND &&
+				global_focus.fadeWindow->xwayland().ctx == server->ctx.get())
+				global_focus.fadeWindow = nullptr;
+
+			if (global_focus.cursor &&
+				global_focus.cursor->getCtx() == server->ctx.get())
+				global_focus.cursor = nullptr;
+
+			wlserver_lock();
+			std::erase_if(pollfds, [=](const auto& other){
+				return other.fd == XConnectionNumber( server->ctx->dpy );
+			});
+			wlserver_destroy_xwayland_server(server);
+			wlserver_unlock();
+
+			focusDirty = true;
+		}
+	}
 	if (ev->atom == ctx->atoms.wineHwndStyle)
 	{
 		steamcompmgr_win_t * w = find_win(ctx, ev->window);
@@ -5615,7 +5705,7 @@ xwayland_ctx_t g_ctx;
 
 static bool setup_error_handlers = false;
 
-void init_xwayland_ctx(gamescope_xwayland_server_t *xwayland_server)
+void init_xwayland_ctx(uint32_t serverId, gamescope_xwayland_server_t *xwayland_server)
 {
 	assert(!xwayland_server->ctx);
 	xwayland_server->ctx = std::make_unique<xwayland_ctx_t>();
@@ -5824,6 +5914,10 @@ void init_xwayland_ctx(gamescope_xwayland_server_t *xwayland_server)
 	ctx->atoms.gamescopeColorShaperLut[DRM_SCREEN_TYPE_INTERNAL] = XInternAtom( ctx->dpy, "GAMESCOPE_COLOR_SHAPERLUT", false );
 	ctx->atoms.gamescopeColorShaperLut[DRM_SCREEN_TYPE_EXTERNAL] = XInternAtom( ctx->dpy, "GAMESCOPE_COLOR_SHAPERLUT_EXTERNAL", false );
 
+	ctx->atoms.gamescopeCreateXWaylandServer = XInternAtom( ctx->dpy, "GAMESCOPE_CREATE_XWAYLAND_SERVER", false );
+	ctx->atoms.gamescopeCreateXWaylandServerFeedback = XInternAtom( ctx->dpy, "GAMESCOPE_CREATE_XWAYLAND_SERVER_FEEDBACK", false );
+	ctx->atoms.gamescopeDestroyXWaylandServer = XInternAtom( ctx->dpy, "GAMESCOPE_DESTROY_XWAYLAND_SERVER", false );
+
 	ctx->atoms.wineHwndStyle = XInternAtom( ctx->dpy, "_WINE_HWND_STYLE", false );
 	ctx->atoms.wineHwndStyleEx = XInternAtom( ctx->dpy, "_WINE_HWND_EXSTYLE", false );
 
@@ -5833,6 +5927,9 @@ void init_xwayland_ctx(gamescope_xwayland_server_t *xwayland_server)
 	ctx->allDamage = None;
 	ctx->clipChanged = true;
 
+	XChangeProperty(ctx->dpy, ctx->root, ctx->atoms.gamescopeXwaylandServerId, XA_CARDINAL, 32, PropModeReplace,
+		(unsigned char *)&serverId, 1 );
+
 	XGrabServer(ctx->dpy);
 
 	XCompositeRedirectSubwindows(ctx->dpy, ctx->root, CompositeRedirectManual);
@@ -6097,7 +6194,7 @@ steamcompmgr_main(int argc, char **argv)
 	{
 		gamescope_xwayland_server_t *server = NULL;
 		for (size_t i = 0; (server = wlserver_get_xwayland_server(i)); i++)
-			init_xwayland_ctx(server);
+			init_xwayland_ctx(i, server);
 	}
 
 	gamescope_xwayland_server_t *root_server = wlserver_get_xwayland_server(0);
@@ -6126,7 +6223,6 @@ steamcompmgr_main(int argc, char **argv)
 	std::thread imageWaitThread( imageWaitThreadMain );
 	imageWaitThread.detach();
 
-	std::vector<pollfd> pollfds;
 	// EVENT_VBLANK
 	pollfds.push_back(pollfd {
 		.fd = vblankFD,
@@ -6147,10 +6243,6 @@ steamcompmgr_main(int argc, char **argv)
 				.events = POLLIN,
 			});
 
-			uint32_t serverId = static_cast<uint32_t>(i);
-			XChangeProperty(server->ctx->dpy, server->ctx->root, server->ctx->atoms.gamescopeXwaylandServerId, XA_CARDINAL, 32, PropModeReplace,
-				(unsigned char *)&serverId, 1 );
-
 			server->ctx->force_windows_fullscreen = bForceWindowsFullscreen;
 		}
 	}

src/steamcompmgr.hpp

@@ -74,6 +74,8 @@ public:
 
 	void undirty() { getTexture(); }
 
+	xwayland_ctx_t *getCtx() const { return m_ctx; }
+
 private:
 	void warp(int x, int y);
 	void checkSuspension();
@@ -134,4 +136,6 @@ struct wlserver_x11_surface_info *lookup_x11_surface_info_from_xid( gamescope_xw
 extern uint64_t g_SteamCompMgrVBlankTime;
 extern pid_t focusWindow_pid;
 
+void init_xwayland_ctx(uint32_t serverId, gamescope_xwayland_server_t *xwayland_server);
+
 extern int g_nAsyncFlipsEnabled;

src/wlserver.cpp

@@ -1731,3 +1731,27 @@ std::vector<ResListEntry_t> wlserver_xdg_commit_queue()
 	}
 	return commits;
 }
+
+uint32_t wlserver_make_new_xwayland_server()
+{
+	assert( wlserver_is_lock_held() );
+
+	auto& server = wlserver.wlr.xwayland_servers.emplace_back(std::make_unique<gamescope_xwayland_server_t>(wlserver.display));
+
+	while (!server->is_xwayland_ready()) {
+		wl_display_flush_clients(wlserver.display);
+		if (wl_event_loop_dispatch(wlserver.event_loop, -1) < 0) {
+			wl_log.errorf("wl_event_loop_dispatch failed\n");
+			return ~0u;
+		}
+	}
+
+	return uint32_t(wlserver.wlr.xwayland_servers.size() - 1);
+}
+
+void wlserver_destroy_xwayland_server(gamescope_xwayland_server_t *server)
+{
+	assert( wlserver_is_lock_held() );
+
+	std::erase_if(wlserver.wlr.xwayland_servers, [=](const auto& other) { return other.get() == server; });
+}

src/wlserver.hpp

@@ -243,3 +243,6 @@ void wlserver_set_xwayland_server_mode( size_t idx, int w, int h, int refresh );
 extern std::atomic<bool> g_bPendingTouchMovement;
 
 void wlserver_open_steam_menu( bool qam );
+
+uint32_t wlserver_make_new_xwayland_server();
+void wlserver_destroy_xwayland_server(gamescope_xwayland_server_t *server);

src/xwayland_ctx.hpp

@@ -195,6 +195,10 @@ struct xwayland_ctx_t
 		Atom gamescopeColorLut3D[DRM_SCREEN_TYPE_COUNT];
 		Atom gamescopeColorShaperLut[DRM_SCREEN_TYPE_COUNT];
 
+		Atom gamescopeCreateXWaylandServer;
+		Atom gamescopeCreateXWaylandServerFeedback;
+		Atom gamescopeDestroyXWaylandServer;
+
 		Atom wineHwndStyle;
 		Atom wineHwndStyleEx;
 	} atoms;