nv2a: Ignore alpha when processing PVIDEO color keying

Discards the alpha value passed to the color key register, matching tested
hardware behavior.

Fixes #2421

Tests: https://github.com/abaire/nxdk_pgraph_tests/blob/c64f5af5f7c3a890f1a00a927c6b72c2656ea025/src/tests/pvideo_tests.cpp#L761

Note: PVIDEO tests do not produce artifacts so they must be compared manually
against HW. I have verified that they are identical for the subset of surface
formats supported by xemu.

Author: abaire

hw/xbox/nv2a/nv2a_regs.h

@@ -696,7 +696,6 @@
 #   define NV_PVIDEO_COLOR_KEY_RED                            0x00FF0000
 #   define NV_PVIDEO_COLOR_KEY_GREEN                          0x0000FF00
 #   define NV_PVIDEO_COLOR_KEY_BLUE                           0x000000FF
-#   define NV_PVIDEO_COLOR_KEY_ALPHA                          0xFF000000
 
 
 #define NV_PTIMER_INTR_0                                 0x00000100

hw/xbox/nv2a/pgraph/gl/display.c

@@ -60,22 +60,22 @@ void pgraph_gl_init_display(NV2AState *d)
         "uniform vec4 pvideo_pos;\n"
         "uniform vec3 pvideo_scale;\n"
         "uniform bool pvideo_color_key_enable;\n"
-        "uniform vec4 pvideo_color_key;\n"
+        "uniform vec3 pvideo_color_key;\n"
         "uniform vec2 display_size;\n"
         "uniform float line_offset;\n"
         "layout(location = 0) out vec4 out_Color;\n"
         "void main()\n"
         "{\n"
         "    vec2 texCoord = gl_FragCoord.xy/display_size;\n"
         "    float rel = display_size.y/textureSize(tex, 0).y/line_offset;\n"
-        "    texCoord.y = rel*(1.0f - texCoord.y);"
+        "    texCoord.y = rel*(1.0f - texCoord.y);\n"
         "    out_Color.rgba = texture(tex, texCoord);\n"
         "    if (pvideo_enable) {\n"
         "        vec2 screenCoord = gl_FragCoord.xy - 0.5;\n"
         "        vec4 output_region = vec4(pvideo_pos.xy, pvideo_pos.xy + pvideo_pos.zw);\n"
         "        bvec4 clip = bvec4(lessThan(screenCoord, output_region.xy),\n"
         "                           greaterThan(screenCoord, output_region.zw));\n"
-        "        if (!any(clip) && (!pvideo_color_key_enable || out_Color.rgba == pvideo_color_key)) {\n"
+        "        if (!any(clip) && (!pvideo_color_key_enable || out_Color.rgb == pvideo_color_key)) {\n"
         "            vec2 out_xy = (screenCoord - pvideo_pos.xy) * pvideo_scale.z;\n"
         "            vec2 in_st = (pvideo_in_pos + out_xy * pvideo_scale.xy) / textureSize(pvideo_tex, 0);\n"
         "            in_st.y *= -1.0;\n"
@@ -242,15 +242,11 @@ static void render_display_pvideo_overlay(NV2AState *d)
     glUniform1ui(r->disp_rndr.pvideo_color_key_enable_loc,
                  color_key_enabled);
 
-    // TODO: Verify that masking off the top byte is correct.
-    // SeaBlade sets a color key of 0x80000000 but the texture passed into the
-    // shader is cleared to 0 alpha.
-    unsigned int color_key = d->pvideo.regs[NV_PVIDEO_COLOR_KEY] & 0xFFFFFF;
-    glUniform4f(r->disp_rndr.pvideo_color_key_loc,
+    unsigned int color_key = d->pvideo.regs[NV_PVIDEO_COLOR_KEY] & 0xFFFFF;
+    glUniform3f(r->disp_rndr.pvideo_color_key_loc,
                 GET_MASK(color_key, NV_PVIDEO_COLOR_KEY_RED) / 255.0,
                 GET_MASK(color_key, NV_PVIDEO_COLOR_KEY_GREEN) / 255.0,
-                GET_MASK(color_key, NV_PVIDEO_COLOR_KEY_BLUE) / 255.0,
-                GET_MASK(color_key, NV_PVIDEO_COLOR_KEY_ALPHA) / 255.0);
+                GET_MASK(color_key, NV_PVIDEO_COLOR_KEY_BLUE) / 255.0);
 
     assert(offset + in_pitch * in_height <= limit);
     hwaddr end = base + offset + in_pitch * in_height;

hw/xbox/nv2a/pgraph/pgraph.c

@@ -3063,7 +3063,7 @@ void pgraph_get_clear_color(PGRAPHState *pg, float rgba[4])
         *b = 1.0f;
         fprintf(stderr, "CLEAR_SURFACE for color_format 0x%x unsupported",
                 pg->surface_shape.color_format);
-        assert(false);
+        assert(!"CLEAR_SURFACE not supported for selected surface format");
         break;
     }
 
@@ -3077,7 +3077,7 @@ void pgraph_get_clear_color(PGRAPHState *pg, float rgba[4])
     case NV097_SET_SURFACE_FORMAT_COLOR_LE_X1A7R8G8B8_Z1A7R8G8B8:
     case NV097_SET_SURFACE_FORMAT_COLOR_LE_X1A7R8G8B8_O1A7R8G8B8:
         *a = ((clear_color >> 24) & 0x7F) / 127.0f;
-        assert(false); /* Untested */
+        assert(!"CLEAR_SURFACE handling for LE_X1A7R8G8B8_Z1A7R8G8B8 and LE_X1A7R8G8B8_O1A7R8G8B8 is untested"); /* Untested */
         break;
     case NV097_SET_SURFACE_FORMAT_COLOR_LE_A8R8G8B8:
         *a = ((clear_color >> 24) & 0xFF) / 255.0f;

hw/xbox/nv2a/pgraph/vk/display.c

@@ -213,22 +213,22 @@ static const char *display_frag_glsl =
     "    vec4 pvideo_pos;\n"
     "    vec4 pvideo_scale;\n"
     "    bool pvideo_color_key_enable;\n"
-    "    vec4 pvideo_color_key;\n"
+    "    vec3 pvideo_color_key;\n"
     "};\n"
     "layout(location = 0) out vec4 out_Color;\n"
     "void main()\n"
     "{\n"
     "    vec2 tex_coord = gl_FragCoord.xy/display_size;\n"
     "    float rel = display_size.y/textureSize(tex, 0).y/line_offset;\n"
-    "    tex_coord.y = 1 + rel*(tex_coord.y - 1);"
+    "    tex_coord.y = 1 + rel*(tex_coord.y - 1);\n"
     "    tex_coord.y = 1 - tex_coord.y;\n" // GL compat
     "    out_Color.rgba = texture(tex, tex_coord);\n"
     "    if (pvideo_enable) {\n"
     "        vec2 screen_coord = vec2(gl_FragCoord.x, display_size.y - gl_FragCoord.y) * pvideo_scale.z;\n"
     "        vec4 output_region = vec4(pvideo_pos.xy, pvideo_pos.xy + pvideo_pos.zw);\n"
     "        bvec4 clip = bvec4(lessThan(screen_coord, output_region.xy),\n"
     "                           greaterThan(screen_coord, output_region.zw));\n"
-    "        if (!any(clip) && (!pvideo_color_key_enable || out_Color.rgba == pvideo_color_key)) {\n"
+    "        if (!any(clip) && (!pvideo_color_key_enable || out_Color.rgb == pvideo_color_key)) {\n"
     "            vec2 out_xy = screen_coord - pvideo_pos.xy;\n"
     "            vec2 in_st = (pvideo_in_pos + out_xy * pvideo_scale.xy) / textureSize(pvideo_tex, 0);\n"
     "            out_Color.rgba = texture(pvideo_tex, in_st);\n"
@@ -849,10 +849,8 @@ static PvideoState get_pvideo_state(PGRAPHState *pg)
     state.color_key_enabled =
         GET_MASK(d->pvideo.regs[NV_PVIDEO_FORMAT], NV_PVIDEO_FORMAT_DISPLAY);
 
-    // TODO: Verify that masking off the top byte is correct.
-    // SeaBlade sets a color key of 0x80000000 but the texture passed into the
-    // shader is cleared to 0 alpha.
-    state.color_key = d->pvideo.regs[NV_PVIDEO_COLOR_KEY] & 0xFFFFFF;
+    // Note: PVIDEO color keying ignores alpha.
+    state.color_key = d->pvideo.regs[NV_PVIDEO_COLOR_KEY] & 0xFFFFF;
 
     assert(state.offset + state.pitch * state.in_height <= state.limit);
     hwaddr end = state.base + state.offset + state.pitch * state.in_height;
@@ -883,12 +881,11 @@ static void update_uniforms(PGRAPHState *pg, SurfaceBinding *surface)
     if (pvideo->enabled) {
         uniform1i(l, uniform_index(l, "pvideo_color_key_enable"),
                   pvideo->color_key_enabled);
-        uniform4f(
+        uniform3f(
             l, uniform_index(l, "pvideo_color_key"),
             GET_MASK(pvideo->color_key, NV_PVIDEO_COLOR_KEY_RED) / 255.0,
             GET_MASK(pvideo->color_key, NV_PVIDEO_COLOR_KEY_GREEN) / 255.0,
-            GET_MASK(pvideo->color_key, NV_PVIDEO_COLOR_KEY_BLUE) / 255.0,
-            GET_MASK(pvideo->color_key, NV_PVIDEO_COLOR_KEY_ALPHA) / 255.0);
+            GET_MASK(pvideo->color_key, NV_PVIDEO_COLOR_KEY_BLUE) / 255.0);
         uniform2f(l, uniform_index(l, "pvideo_in_pos"), pvideo->in_s,
                   pvideo->in_t);
         uniform4f(l, uniform_index(l, "pvideo_pos"), pvideo->out_x,