rendervulkan: Replace flBorderAlpha with nBorderMask of layers

Allows us to fit more layers in push constants. We never use anything other than 0/1 anyway.

src/composite.comp

@@ -22,7 +22,7 @@ uniform layers_t {
     vec2 u_scale[MaxLayers];
     vec2 u_offset[MaxLayers];
     float u_opacity[MaxLayers];
-    float u_borderAlpha[MaxLayers];
+    uint u_borderMask;
     uint u_frameId;
 };
 
@@ -75,8 +75,10 @@ vec4 sampleLayer(sampler2D layerSampler, uint layerIdx, vec2 uv) {
     vec2 texSize = textureSize(layerSampler, 0);
 
     if (coord.x < 0.0f       || coord.y < 0.0f ||
-        coord.x >= texSize.x || coord.y >= texSize.y )
+        coord.x >= texSize.x || coord.y >= texSize.y) {
-        return vec4(0.0f, 0.0f, 0.0f, u_borderAlpha[layerIdx]);
+        float border = (u_borderMask & (1u << layerIdx)) != 0 ? 1.0f : 0.0f;
+        return vec4(0.0f, 0.0f, 0.0f, border);
+    }
 
     return textureLod(layerSampler, coord, 0.0f);
 }

src/rendervulkan.hpp

@@ -44,8 +44,9 @@ struct Composite_t
 		vec2_t vScale[k_nMaxLayers];
 		vec2_t vOffset[k_nMaxLayers];
 		float flOpacity[k_nMaxLayers];
-		float flBorderAlpha[k_nMaxLayers];
+		uint32_t nBorderMask;
 	} data;
+
 };
 
 #include "drm.hpp"

src/steamcompmgr.cpp

@@ -1134,8 +1134,6 @@ void MouseCursor::paint(win *window, struct Composite_t *pComposite,
 	pComposite->data.vOffset[ curLayer ].x = -scaledX;
 	pComposite->data.vOffset[ curLayer ].y = -scaledY;
 
-	pComposite->data.flBorderAlpha[ curLayer ] = 0.0f;
-
 	pPipeline->layerBindings[ curLayer ].surfaceWidth = m_width;
 	pPipeline->layerBindings[ curLayer ].surfaceHeight = m_height;
 
@@ -1174,6 +1172,8 @@ paint_cached_base_layer(const std::shared_ptr<commit_t>& commit, const BaseLayer
 	pPipeline->layerBindings[ curLayer ].fbid = commit->fb_id;
 	pPipeline->layerBindings[ curLayer ].bFilter = true;
 
+	pComposite->data.nBorderMask |= (1u << curLayer);
+
 	pComposite->nLayerCount++;
 }
 
@@ -1279,8 +1279,6 @@ paint_window(Display *dpy, win *w, win *scaleW, struct Composite_t *pComposite,
 	{
 		pComposite->data.vOffset[ curLayer ].x = -drawXOffset;
 		pComposite->data.vOffset[ curLayer ].y = -drawYOffset;
-
-		pComposite->data.flBorderAlpha[ curLayer ] = 0.0f;
 	}
 	else if (notificationMode)
 	{
@@ -1297,15 +1295,13 @@ paint_window(Display *dpy, win *w, win *scaleW, struct Composite_t *pComposite,
 
 		pComposite->data.vOffset[ curLayer ].x = (currentOutputWidth - xOffset - width) * -1.0f;
 		pComposite->data.vOffset[ curLayer ].y = (currentOutputHeight - yOffset - height) * -1.0f;
-
-		pComposite->data.flBorderAlpha[ curLayer ] = 0.0f;
 	}
 	else
 	{
 		pComposite->data.vOffset[ curLayer ].x = -drawXOffset;
 		pComposite->data.vOffset[ curLayer ].y = -drawYOffset;
 
-		pComposite->data.flBorderAlpha[ curLayer ] = 1.0f;
+		pComposite->data.nBorderMask |= (1u << curLayer);
 	}
 
 	pPipeline->layerBindings[ curLayer ].surfaceWidth = w->a.width;