The homemade chroma key filter I have developed cannot change the color of the commonly used green and blue background colors.

[JaredWei]

Hello everyone, I refer to the OBS Studio 20.1.0 documentation written by the forum and the chroma_key_filter.effect on github. In the recent self-made obs-studio filter, I have a problem. I don’t think how to solve it. I would like to ask you here, I hope I can get some suggestions or answers.

The first problem is that when the power point presentation is captured, the background is set to red (Figure 1), and then I can get the edge of the text with my own edge detection kernel, but the usual green(or blue) color will not show the effect (Figure 2)? (The OBS-studio resource uses a white background for the convenience of seeing the effect)

Figure 1.

Figure 2.

The second problem is that the filter has changed to other kernel. On the contrary, the parameterized horizontal scroll bar have no effect. Is it necessary to adjust the position to make the fullmask work (Figure 3)?

Figure 3.

The code show is as follows(Actually, I use the HLSL or GLSL language. I don't know why the code is added in the C++ language):

uniform float4x4 ViewProj;
uniform texture2d image;

uniform float4x4 yuv_mat = { 0.182586,  0.614231,  0.062007, 0.062745,
                            -0.100644, -0.338572,  0.439216, 0.501961,
                             0.439216, -0.398942, -0.040274, 0.501961,
                             0.000000,  0.000000,  0.000000, 1.000000};

uniform float4 color;
uniform float contrast;
uniform float brightness;
uniform float gamma;

uniform float2 chroma_key;
uniform float4 key_rgb;
uniform float2 pixel_size;
uniform float similarity;
uniform float smoothness;
uniform float spill;

// 3x3 kernel for convolution edge detection.
uniform float3 kernel1  = { -1.0, -1.0, -1.0};
uniform float3 kernel2    = {-1.0, 8.0, -1.0};
uniform float3 kernel3    = {-1.0, -1.0, -1.0 };

sampler_state textureSampler {
    Filter    = Linear;
    AddressU  = Clamp;
    AddressV  = Clamp;
};

struct VertData {
    float4 pos : POSITION;
    float2 uv  : TEXCOORD0; //Texture coordinates.
};

VertData VSDefault(VertData v_in)
{
    VertData vert_out;
    vert_out.pos = mul(float4(v_in.pos.xyz, 1.0), ViewProj);
    vert_out.uv  = v_in.uv;
    return vert_out;
}

float4 CalcColor(float4 rgba)
{
    return float4(pow(rgba.rgb, float3(gamma, gamma, gamma)) * contrast + brightness, rgba.a);
}

float GetChromaDist(float3 rgb)
{
    float4 yuvx = mul(float4(rgb.rgb, 1.0), yuv_mat); //rgb to yuv
    return distance(chroma_key, yuvx.yz);             // Take the distance scalar value of two vectors.
}

float4 SampleTexture(float2 uv)
{
    return image.Sample(textureSampler, uv); 
}

//  3x3 filter of Edge detection
float GetEdgeDetectionFilteredChromaDist(float3 rgb, float2 texCoord)
{
    float distVal = SampleTexture(-texCoord-pixel_size).rgb * kernel1[0];                        // Top left
    distVal += SampleTexture(texCoord-pixel_size).rgb * kernel1[1];                             // Top center
    distVal += SampleTexture(texCoord-float2(pixel_size.x, 0.0)).rgb * kernel1[2];              // Top right

    distVal += SampleTexture(texCoord-float2(pixel_size.x, -pixel_size.y)).rgb * kernel2[0];    // Middle left
    distVal += SampleTexture(texCoord-float2(0.0, pixel_size.y)).rgb * kernel2[1];              // Current pixel
    distVal += SampleTexture(texCoord+float2(0.0, pixel_size.y)).rgb * kernel2[2];              // Middle right

    distVal += SampleTexture(texCoord+float2(pixel_size.x, -pixel_size.y)).rgb * kernel3[0];    // Bottom left
    distVal += SampleTexture(texCoord+float2(pixel_size.x, 0.0)).rgb * kernel3[1];              // Bottom center
    distVal += SampleTexture(texCoord+pixel_size).rgb * kernel3[2];                             // Bottom right

    return distVal;
}


float4 ProcessChromaKey(float4 rgba, VertData v_in)
{
    float chromaDist = GetEdgeDetectionFilteredChromaDist(rgba.rgb, v_in.uv);//Edge detection filter function.

    float baseMask = chromaDist - similarity;
    float fullMask = pow(saturate(baseMask / smoothness), 1.5);
    float spillVal = pow(saturate(baseMask / spill), 1.5);

    rgba.rgba *= color;
    rgba.a *= fullMask;

    float desat = (rgba.r * 0.2126 + rgba.g * 0.7152 + rgba.b * 0.0722);
    rgba.rgb = saturate(float3(desat, desat, desat)) * (1.0 - spillVal) + rgba.rgb * spillVal;

    return CalcColor(rgba);
}

float4 PSChromaKeyRGBA(VertData v_in) : TARGET
{
    float4 rgba = image.Sample(textureSampler, v_in.uv);
    return ProcessChromaKey(rgba, v_in);
}

technique Draw
{
    pass
    {
        vertex_shader = VSDefault(v_in);
        pixel_shader  = PSChromaKeyRGBA(v_in);
    }
}

Thank you