From 31a07134559f63811eaee3ce6678881c1b94da58 Mon Sep 17 00:00:00 2001
From: Simon Ser <contact@emersion.fr>
Date: Tue, 19 May 2020 11:26:52 +0200
Subject: [PATCH] Copy commit queue in check_new_wayland_res

This will allow us to call wlr_buffer_lock/unlock in import_commit without
causing a deadlock.
---
 src/steamcompmgr.cpp | 126 +++++++++++++++++++++----------------------
 1 file changed, 60 insertions(+), 66 deletions(-)

diff --git a/src/steamcompmgr.cpp b/src/steamcompmgr.cpp
index f44ee31..8457c8f 100644
--- a/src/steamcompmgr.cpp
+++ b/src/steamcompmgr.cpp
@@ -2362,79 +2362,73 @@ void handle_done_commits( void )
 
 void check_new_wayland_res( void )
 {
-	// We need to release buffers without having a commit queue lock, otherwise
-	// we won't be able to lock wlserver without a deadlock
-	std::vector<struct wlr_buffer *> release_queue;
-
+	// When importing buffer, we'll potentially need to perform operations with
+	// a wlserver lock (e.g. wlr_buffer_lock). We can't do this with a
+	// wayland_commit_queue lock because that causes deadlocks.
+	std::vector<ResListEntry_t> tmp_queue;
 	{
 		std::lock_guard<std::mutex> lock( wayland_commit_lock );
-
-		for ( uint32_t i = 0; i < wayland_commit_queue.size(); i++ )
-		{
-			struct wlr_buffer *buf = wayland_commit_queue[ i ].buf;
-
-			win	*w = find_win( wayland_commit_queue[ i ].surf );
-
-			if ( w == nullptr )
-			{
-				release_queue.push_back( buf );
-				fprintf (stderr, "waylandres but no win\n");
-				continue;
-			}
-
-			struct wlr_dmabuf_attributes dmabuf = {};
-			bool result = False;
-			if ( wlr_buffer_get_dmabuf( buf, &dmabuf ) ) {
-				result = true;
-				for ( int i = 0; i < dmabuf.n_planes; i++ ) {
-					dmabuf.fd[i] = dup( dmabuf.fd[i] );
-					assert( dmabuf.fd[i] >= 0 );
-				}
-			} else {
-				struct wlr_client_buffer *client_buf = (struct wlr_client_buffer *) buf;
-				result = wlr_texture_to_dmabuf( client_buf->texture, &dmabuf );
-			}
-			assert( result == true );
-
-			commit_t newCommit = {};
-			bool bSuccess = import_commit( buf, &dmabuf, newCommit );
-			wlr_dmabuf_attributes_finish( &dmabuf );
-
-			if ( bSuccess == true )
-			{
-				newCommit.commitID = ++maxCommmitID;
-				w->commit_queue.push_back( newCommit );
-			}
-
-			int fence = vulkan_get_texture_fence( newCommit.vulkanTex );
-
-			if ( fence < 0 )
-			{
-				fprintf (stderr, "no fence for texture\n");
-				continue;
-			}
-
-			gpuvis_trace_printf( "pushing wait for commit %lu\n", newCommit.commitID );
-			{
-				std::unique_lock< std::mutex > lock( waitListLock );
-
-				waitList.push_back( std::make_pair( fence, newCommit.commitID ) );
-			}
-
-			// Wake up commit wait thread if chilling
-			waitListSem.signal();
-		}
-
+		tmp_queue = wayland_commit_queue;
 		wayland_commit_queue.clear();
 	}
 
-	wlserver_lock();
-	for ( uint32_t i = 0; i < release_queue.size(); i++ )
+	for ( uint32_t i = 0; i < tmp_queue.size(); i++ )
 	{
-		wlr_buffer_unlock( release_queue[ i ] );
+		struct wlr_buffer *buf = tmp_queue[ i ].buf;
+
+		win	*w = find_win( tmp_queue[ i ].surf );
+
+		if ( w == nullptr )
+		{
+			wlserver_lock();
+			wlr_buffer_unlock( buf );
+			wlserver_unlock();
+			fprintf (stderr, "waylandres but no win\n");
+			continue;
+		}
+
+		struct wlr_dmabuf_attributes dmabuf = {};
+		bool result = False;
+		if ( wlr_buffer_get_dmabuf( buf, &dmabuf ) ) {
+			result = true;
+			for ( int i = 0; i < dmabuf.n_planes; i++ ) {
+				dmabuf.fd[i] = dup( dmabuf.fd[i] );
+				assert( dmabuf.fd[i] >= 0 );
+			}
+		} else {
+			struct wlr_client_buffer *client_buf = (struct wlr_client_buffer *) buf;
+			result = wlr_texture_to_dmabuf( client_buf->texture, &dmabuf );
+		}
+		assert( result == true );
+
+		commit_t newCommit = {};
+		bool bSuccess = import_commit( buf, &dmabuf, newCommit );
+		wlr_dmabuf_attributes_finish( &dmabuf );
+
+		if ( bSuccess == true )
+		{
+			newCommit.commitID = ++maxCommmitID;
+			w->commit_queue.push_back( newCommit );
+		}
+
+		int fence = vulkan_get_texture_fence( newCommit.vulkanTex );
+
+		if ( fence < 0 )
+		{
+			fprintf (stderr, "no fence for texture\n");
+			continue;
+		}
+
+		gpuvis_trace_printf( "pushing wait for commit %lu\n", newCommit.commitID );
+		{
+			std::unique_lock< std::mutex > lock( waitListLock );
+
+			waitList.push_back( std::make_pair( fence, newCommit.commitID ) );
+		}
+
+		// Wake up commit wait thread if chilling
+		waitListSem.signal();
 	}
-	wlserver_unlock();
-	release_queue.clear();
 }
 
 int