rendervulkan: Replace border color with CLAMP_TO_EDGE + bounds check

We can't use normal border colors if we are doing linear filtering as we will sample slightly outside on the edges and blend to the border color.

Replace this with a way to push an arbitrary layer alpha and return that in the OOB case, and when sampling, clamp to edge.

This also may be more efficient to avoid sampling OOB a lot of the time in some cases.

Signed-off-by: Joshua Ashton <joshua@froggi.es>

src/composite.comp

@@ -20,19 +20,28 @@ uniform layers_t {
     vec2 u_scale[MaxLayers];
     vec2 u_offset[MaxLayers];
     float u_opacity[MaxLayers];
+    float u_borderAlpha[MaxLayers];
 };
 
 layout(binding = 2) uniform sampler2D s_samplers[MaxLayers];
 
 layout(binding = 6) uniform sampler2D s_ycbcr_samplers[MaxLayers];
 
-vec4 sampleLayer(uint layerIdx, vec2 uv) {
-    vec2 coord = (uv + u_offset[layerIdx]) * u_scale[layerIdx];
+vec4 sampleLayer(sampler2D layerSampler, uint layerIdx, vec2 uv) {
+    vec2 coord = ((uv + u_offset[layerIdx]) * u_scale[layerIdx]);
+    vec2 texSize = textureSize(layerSampler, 0);
 
+    if (coord.x < 0.0f       || coord.y < 0.0f ||
+        coord.x >= texSize.x || coord.y >= texSize.y )
+        return vec4(0.0f, 0.0f, 0.0f, u_borderAlpha[layerIdx]);
+
+    return texture(layerSampler, coord);
+}
+
+vec4 sampleLayer(uint layerIdx, vec2 uv) {
     if ((c_ycbcrMask & (1 << layerIdx)) != 0)
-        return texture(s_ycbcr_samplers[layerIdx], coord);
-    else
-        return texture(s_samplers[layerIdx], coord);
+        return sampleLayer(s_ycbcr_samplers[layerIdx], layerIdx, uv);
+    return sampleLayer(s_samplers[layerIdx], layerIdx, uv);
 }
 
 void main() {

src/rendervulkan.cpp

@@ -1094,13 +1094,6 @@ retry:
 		.pNext = &ycbcrSamplerConversionInfo,
 		.magFilter = VK_FILTER_LINEAR,
 		.minFilter = VK_FILTER_LINEAR,
-		// Using clamp to edge here for now, otherwise we end up with a
-		// 1/2 pixel of green on the left and top of the screen due to
-		// the texel replacement happening before the YCbCr conversion.
-
-		// TODO: Find out why we are even hitting the border in the first place,
-		// maybe there is some weird half-texel stuff somewhere in Gamescope causing this,
-		// I imagine if this was more obvious it would affect games too.
 		.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
 		.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
 		.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK,
@@ -1931,9 +1924,6 @@ bool operator==(const struct VulkanPipeline_t::LayerBinding_t& lhs, struct Vulka
 	if ( lhs.bFilter != rhs.bFilter )
 		return false;
 
-	if ( lhs.bBlackBorder != rhs.bBlackBorder )
-		return false;
-
 	return true;
 }
 
@@ -1946,9 +1936,9 @@ VkSampler vulkan_make_sampler( struct VulkanPipeline_t::LayerBinding_t *pBinding
 		.pNext = nullptr,
 		.magFilter = pBinding->bFilter ? VK_FILTER_LINEAR : VK_FILTER_NEAREST,
 		.minFilter = pBinding->bFilter ? VK_FILTER_LINEAR : VK_FILTER_NEAREST,
-		.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER,
-		.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER,
-		.borderColor = pBinding->bBlackBorder ? VK_BORDER_COLOR_INT_OPAQUE_BLACK : VK_BORDER_COLOR_INT_TRANSPARENT_BLACK,
+		.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
+		.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
+		.borderColor = VK_BORDER_COLOR_INT_TRANSPARENT_BLACK,
 		.unnormalizedCoordinates = VK_TRUE,
 	};
 	
@@ -2005,9 +1995,6 @@ void vulkan_update_descriptor( struct VulkanPipeline_t *pPipeline, int nYCBCRMas
 		else
 		{
 			pTex = g_mapVulkanTextures[ g_emptyTex ];
-
-			// Switch the border to transparent black
-			pPipeline->layerBindings[ i ].bBlackBorder = false;
 		}
 		
 		// First try to look up the sampler in the cache.
@@ -2117,15 +2104,6 @@ bool vulkan_composite( struct Composite_t *pComposite, struct VulkanPipeline_t *
 		}
 	}
 
-	// Sample a bit closer to texel centers in most cases
-	// TODO: probably actually need to apply a general scale/bias to properly
-	// sample from the center in all four corners in all scaling scenarios
-	for ( int i = 0; i < pComposite->nLayerCount; i++ )
-	{
-		pComposite->data.vOffset[ i ].x += 0.5f;
-		pComposite->data.vOffset[ i ].y += 0.5f;
-	}
-
 	*g_output.pCompositeBuffer = pComposite->data;
 	// XXX maybe flush something?
 	

src/rendervulkan.hpp

@@ -24,7 +24,6 @@ struct VulkanPipeline_t
 
 		// These fields below count for the sampler cache
 		bool bFilter;
-		bool bBlackBorder;
 	} layerBindings[ k_nMaxLayers ];
 };
 
@@ -44,6 +43,7 @@ struct Composite_t
 		vec2_t vScale[k_nMaxLayers];
 		vec2_t vOffset[k_nMaxLayers];
 		float flOpacity[k_nMaxLayers];
+		float flBorderAlpha[k_nMaxLayers];
 	} data;
 };
 

src/steamcompmgr.cpp

@@ -1001,6 +1001,8 @@ void MouseCursor::paint(win *window, struct Composite_t *pComposite,
 	pComposite->data.vOffset[ curLayer ].x = -scaledX;
 	pComposite->data.vOffset[ curLayer ].y = -scaledY;
 
+	pComposite->data.flBorderAlpha[ curLayer ] = 0.0f;
+
 	pPipeline->layerBindings[ curLayer ].surfaceWidth = m_width;
 	pPipeline->layerBindings[ curLayer ].surfaceHeight = m_height;
 
@@ -1011,7 +1013,6 @@ void MouseCursor::paint(win *window, struct Composite_t *pComposite,
 															  vulkan_texture_get_fbid(m_texture);
 
 	pPipeline->layerBindings[ curLayer ].bFilter = false;
-	pPipeline->layerBindings[ curLayer ].bBlackBorder = false;
 
 	pComposite->nLayerCount += 1;
 }
@@ -1095,11 +1096,15 @@ paint_window (Display *dpy, win *w, struct Composite_t *pComposite,
 
 		pComposite->data.vOffset[ curLayer ].x = (currentOutputWidth - xOffset - width) * -1.0f;
 		pComposite->data.vOffset[ curLayer ].y = (currentOutputHeight - yOffset - height) * -1.0f;
+
+		pComposite->data.flBorderAlpha[ curLayer ] = 0.0f;
 	}
 	else
 	{
 		pComposite->data.vOffset[ curLayer ].x = -drawXOffset;
 		pComposite->data.vOffset[ curLayer ].y = -drawYOffset;
+
+		pComposite->data.flBorderAlpha[ curLayer ] = 1.0f;
 	}
 
 	pPipeline->layerBindings[ curLayer ].surfaceWidth = w->a.width;
@@ -1116,7 +1121,6 @@ paint_window (Display *dpy, win *w, struct Composite_t *pComposite,
 	pPipeline->layerBindings[ curLayer ].fbid = lastCommit.fb_id;
 
 	pPipeline->layerBindings[ curLayer ].bFilter = w->isOverlay ? true : g_bFilterGameWindow;
-	pPipeline->layerBindings[ curLayer ].bBlackBorder = notificationMode ? false : true;
 
 	pComposite->nLayerCount += 1;
 }