From 3e1ebd724135a90a14391f096606560f85333a75 Mon Sep 17 00:00:00 2001 From: Joshua Ashton Date: Fri, 27 Aug 2021 03:50:08 +0100 Subject: [PATCH] rendervulkan: Replace border color with CLAMP_TO_EDGE + bounds check We can't use normal border colors if we are doing linear filtering as we will sample slightly outside on the edges and blend to the border color. Replace this with a way to push an arbitrary layer alpha and return that in the OOB case, and when sampling, clamp to edge. This also may be more efficient to avoid sampling OOB a lot of the time in some cases. Signed-off-by: Joshua Ashton --- src/composite.comp | 19 ++++++++++++++----- src/rendervulkan.cpp | 28 +++------------------------- src/rendervulkan.hpp | 2 +- src/steamcompmgr.cpp | 8 ++++++-- 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/composite.comp b/src/composite.comp index 332ec97..09b2057 100644 --- a/src/composite.comp +++ b/src/composite.comp @@ -20,19 +20,28 @@ uniform layers_t { vec2 u_scale[MaxLayers]; vec2 u_offset[MaxLayers]; float u_opacity[MaxLayers]; + float u_borderAlpha[MaxLayers]; }; layout(binding = 2) uniform sampler2D s_samplers[MaxLayers]; layout(binding = 6) uniform sampler2D s_ycbcr_samplers[MaxLayers]; -vec4 sampleLayer(uint layerIdx, vec2 uv) { - vec2 coord = (uv + u_offset[layerIdx]) * u_scale[layerIdx]; +vec4 sampleLayer(sampler2D layerSampler, uint layerIdx, vec2 uv) { + vec2 coord = ((uv + u_offset[layerIdx]) * u_scale[layerIdx]); + vec2 texSize = textureSize(layerSampler, 0); + if (coord.x < 0.0f || coord.y < 0.0f || + coord.x >= texSize.x || coord.y >= texSize.y ) + return vec4(0.0f, 0.0f, 0.0f, u_borderAlpha[layerIdx]); + + return texture(layerSampler, coord); +} + +vec4 sampleLayer(uint layerIdx, vec2 uv) { if ((c_ycbcrMask & (1 << layerIdx)) != 0) - return texture(s_ycbcr_samplers[layerIdx], coord); - else - return texture(s_samplers[layerIdx], coord); + return sampleLayer(s_ycbcr_samplers[layerIdx], layerIdx, uv); + return sampleLayer(s_samplers[layerIdx], layerIdx, uv); } void main() { diff --git a/src/rendervulkan.cpp b/src/rendervulkan.cpp index 9ac4865..ea03751 100644 --- a/src/rendervulkan.cpp +++ b/src/rendervulkan.cpp @@ -1094,13 +1094,6 @@ retry: .pNext = &ycbcrSamplerConversionInfo, .magFilter = VK_FILTER_LINEAR, .minFilter = VK_FILTER_LINEAR, - // Using clamp to edge here for now, otherwise we end up with a - // 1/2 pixel of green on the left and top of the screen due to - // the texel replacement happening before the YCbCr conversion. - - // TODO: Find out why we are even hitting the border in the first place, - // maybe there is some weird half-texel stuff somewhere in Gamescope causing this, - // I imagine if this was more obvious it would affect games too. .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, .borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK, @@ -1931,9 +1924,6 @@ bool operator==(const struct VulkanPipeline_t::LayerBinding_t& lhs, struct Vulka if ( lhs.bFilter != rhs.bFilter ) return false; - if ( lhs.bBlackBorder != rhs.bBlackBorder ) - return false; - return true; } @@ -1946,9 +1936,9 @@ VkSampler vulkan_make_sampler( struct VulkanPipeline_t::LayerBinding_t *pBinding .pNext = nullptr, .magFilter = pBinding->bFilter ? VK_FILTER_LINEAR : VK_FILTER_NEAREST, .minFilter = pBinding->bFilter ? VK_FILTER_LINEAR : VK_FILTER_NEAREST, - .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, - .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, - .borderColor = pBinding->bBlackBorder ? VK_BORDER_COLOR_INT_OPAQUE_BLACK : VK_BORDER_COLOR_INT_TRANSPARENT_BLACK, + .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, + .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, + .borderColor = VK_BORDER_COLOR_INT_TRANSPARENT_BLACK, .unnormalizedCoordinates = VK_TRUE, }; @@ -2005,9 +1995,6 @@ void vulkan_update_descriptor( struct VulkanPipeline_t *pPipeline, int nYCBCRMas else { pTex = g_mapVulkanTextures[ g_emptyTex ]; - - // Switch the border to transparent black - pPipeline->layerBindings[ i ].bBlackBorder = false; } // First try to look up the sampler in the cache. @@ -2117,15 +2104,6 @@ bool vulkan_composite( struct Composite_t *pComposite, struct VulkanPipeline_t * } } - // Sample a bit closer to texel centers in most cases - // TODO: probably actually need to apply a general scale/bias to properly - // sample from the center in all four corners in all scaling scenarios - for ( int i = 0; i < pComposite->nLayerCount; i++ ) - { - pComposite->data.vOffset[ i ].x += 0.5f; - pComposite->data.vOffset[ i ].y += 0.5f; - } - *g_output.pCompositeBuffer = pComposite->data; // XXX maybe flush something? diff --git a/src/rendervulkan.hpp b/src/rendervulkan.hpp index ea64db5..cbb8833 100644 --- a/src/rendervulkan.hpp +++ b/src/rendervulkan.hpp @@ -24,7 +24,6 @@ struct VulkanPipeline_t // These fields below count for the sampler cache bool bFilter; - bool bBlackBorder; } layerBindings[ k_nMaxLayers ]; }; @@ -44,6 +43,7 @@ struct Composite_t vec2_t vScale[k_nMaxLayers]; vec2_t vOffset[k_nMaxLayers]; float flOpacity[k_nMaxLayers]; + float flBorderAlpha[k_nMaxLayers]; } data; }; diff --git a/src/steamcompmgr.cpp b/src/steamcompmgr.cpp index 920856a..fb418f6 100644 --- a/src/steamcompmgr.cpp +++ b/src/steamcompmgr.cpp @@ -1001,6 +1001,8 @@ void MouseCursor::paint(win *window, struct Composite_t *pComposite, pComposite->data.vOffset[ curLayer ].x = -scaledX; pComposite->data.vOffset[ curLayer ].y = -scaledY; + pComposite->data.flBorderAlpha[ curLayer ] = 0.0f; + pPipeline->layerBindings[ curLayer ].surfaceWidth = m_width; pPipeline->layerBindings[ curLayer ].surfaceHeight = m_height; @@ -1011,7 +1013,6 @@ void MouseCursor::paint(win *window, struct Composite_t *pComposite, vulkan_texture_get_fbid(m_texture); pPipeline->layerBindings[ curLayer ].bFilter = false; - pPipeline->layerBindings[ curLayer ].bBlackBorder = false; pComposite->nLayerCount += 1; } @@ -1095,11 +1096,15 @@ paint_window (Display *dpy, win *w, struct Composite_t *pComposite, pComposite->data.vOffset[ curLayer ].x = (currentOutputWidth - xOffset - width) * -1.0f; pComposite->data.vOffset[ curLayer ].y = (currentOutputHeight - yOffset - height) * -1.0f; + + pComposite->data.flBorderAlpha[ curLayer ] = 0.0f; } else { pComposite->data.vOffset[ curLayer ].x = -drawXOffset; pComposite->data.vOffset[ curLayer ].y = -drawYOffset; + + pComposite->data.flBorderAlpha[ curLayer ] = 1.0f; } pPipeline->layerBindings[ curLayer ].surfaceWidth = w->a.width; @@ -1116,7 +1121,6 @@ paint_window (Display *dpy, win *w, struct Composite_t *pComposite, pPipeline->layerBindings[ curLayer ].fbid = lastCommit.fb_id; pPipeline->layerBindings[ curLayer ].bFilter = w->isOverlay ? true : g_bFilterGameWindow; - pPipeline->layerBindings[ curLayer ].bBlackBorder = notificationMode ? false : true; pComposite->nLayerCount += 1; }