GPU/Rasterizer: Corrected the stencil implementation.

Verified the behavior with hardware tests.

src/video_core/pica.h

@@ -441,8 +441,12 @@ struct Regs {
     };
 
     enum class StencilAction : u32 {
-        Keep = 0,
+        Keep      = 0,
-        Xor  = 5,
+
+        Replace   = 2,
+        Increment = 3,
+        Decrement = 4,
+        Invert    = 5
     };
 
     struct {
@@ -481,23 +485,29 @@ struct Regs {
 
         struct {
             union {
+                // Raw value of this register
+                u32 raw_func;
+
                 // If true, enable stencil testing
                 BitField< 0, 1, u32> enable;
 
                 // Comparison operation for stencil testing
                 BitField< 4, 3, CompareFunc> func;
 
-                // Value to calculate the new stencil value from
+                // Mask used to control writing to the stencil buffer
-                BitField< 8, 8, u32> replacement_value;
+                BitField< 8, 8, u32> write_mask;
 
                 // Value to compare against for stencil testing
                 BitField<16, 8, u32> reference_value;
 
                 // Mask to apply on stencil test inputs
-                BitField<24, 8, u32> mask;
+                BitField<24, 8, u32> input_mask;
             };
 
             union {
+                // Raw value of this register
+                u32 raw_op;
+
                 // Action to perform when the stencil test fails
                 BitField< 0, 3, StencilAction> action_stencil_fail;
 

src/video_core/rasterizer.cpp

@@ -215,14 +215,25 @@ static void SetStencil(int x, int y, u8 value) {
     }
 }
 
-// TODO: Should the stencil mask be applied to the "dest" or "ref" operands? Most likely not!
+// TODO: Should the stencil mask be applied to the "old_stencil" or "ref" operands? Most likely not!
-static u8 PerformStencilAction(Regs::StencilAction action, u8 dest, u8 ref) {
+static u8 PerformStencilAction(Regs::StencilAction action, u8 old_stencil, u8 ref) {
     switch (action) {
     case Regs::StencilAction::Keep:
-        return dest;
+        return old_stencil;
 
-    case Regs::StencilAction::Xor:
+    case Regs::StencilAction::Replace:
-        return dest ^ ref;
+        return ref;
+
+    case Regs::StencilAction::Increment:
+        // Saturated increment
+        return std::min<u8>(old_stencil, 254) + 1;
+
+    case Regs::StencilAction::Decrement:
+        // Saturated decrement
+        return std::max<u8>(old_stencil, 1) - 1;
+
+    case Regs::StencilAction::Invert:
+        return ~old_stencil;
 
     default:
         LOG_CRITICAL(HW_GPU, "Unknown stencil action %x", (int)action);
@@ -782,8 +793,8 @@ static void ProcessTriangleInternal(const Shader::OutputVertex& v0,
             u8 old_stencil = 0;
             if (stencil_action_enable) {
                 old_stencil = GetStencil(x >> 4, y >> 4);
-                u8 dest = old_stencil & stencil_test.mask;
+                u8 dest = old_stencil & stencil_test.input_mask;
-                u8 ref = stencil_test.reference_value & stencil_test.mask;
+                u8 ref = stencil_test.reference_value & stencil_test.input_mask;
 
                 bool pass = false;
                 switch (stencil_test.func) {
@@ -821,8 +832,8 @@ static void ProcessTriangleInternal(const Shader::OutputVertex& v0,
                 }
 
                 if (!pass) {
-                    u8 new_stencil = PerformStencilAction(stencil_test.action_stencil_fail, old_stencil, stencil_test.replacement_value);
+                    u8 new_stencil = PerformStencilAction(stencil_test.action_stencil_fail, old_stencil, stencil_test.reference_value);
-                    SetStencil(x >> 4, y >> 4, new_stencil);
+                    SetStencil(x >> 4, y >> 4, (new_stencil & stencil_test.write_mask) | (old_stencil & ~stencil_test.write_mask));
                     continue;
                 }
             }
@@ -873,8 +884,8 @@ static void ProcessTriangleInternal(const Shader::OutputVertex& v0,
 
                 if (!pass) {
                     if (stencil_action_enable) {
-                        u8 new_stencil = PerformStencilAction(stencil_test.action_depth_fail, old_stencil, stencil_test.replacement_value);
+                        u8 new_stencil = PerformStencilAction(stencil_test.action_depth_fail, old_stencil, stencil_test.reference_value);
-                        SetStencil(x >> 4, y >> 4, new_stencil);
+                        SetStencil(x >> 4, y >> 4, (new_stencil & stencil_test.write_mask) | (old_stencil & ~stencil_test.write_mask));
                     }
                     continue;
                 }
@@ -884,8 +895,8 @@ static void ProcessTriangleInternal(const Shader::OutputVertex& v0,
 
                 if (stencil_action_enable) {
                     // TODO: What happens if stencil testing is enabled, but depth testing is not? Will stencil get updated anyway?
-                    u8 new_stencil = PerformStencilAction(stencil_test.action_depth_pass, old_stencil, stencil_test.replacement_value);
+                    u8 new_stencil = PerformStencilAction(stencil_test.action_depth_pass, old_stencil, stencil_test.reference_value);
-                    SetStencil(x >> 4, y >> 4, new_stencil);
+                    SetStencil(x >> 4, y >> 4, (new_stencil & stencil_test.write_mask) | (old_stencil & ~stencil_test.write_mask));
                 }
             }