From 15d2519d3af1389cb2f789a8968fb16de007b702 Mon Sep 17 00:00:00 2001
From: past-due <30942300+past-due@users.noreply.github.com>
Date: Wed, 17 Jul 2024 14:04:42 -0400
Subject: [PATCH] Improve effects shader when fog is enabled

Fix visual artifacts with premultiplied effect textures and improve additive blending when fog is enabled.
---
 data/base/shaders/nolight_instanced.frag    | 5 ++++-
 data/base/shaders/vk/nolight_instanced.frag | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/data/base/shaders/nolight_instanced.frag b/data/base/shaders/nolight_instanced.frag
index eba0e1064d3..a84bc6f1fed 100644
--- a/data/base/shaders/nolight_instanced.frag
+++ b/data/base/shaders/nolight_instanced.frag
@@ -60,7 +60,10 @@ void main()
 		}
 
 		// Return fragment color
-		fragColour = mix(fragColour, vec4(fogColor.xyz, fragColour.w), clamp(fogFactor, 0.0, 1.0));
+		vec3 fogPremultAlphaFactor = mix(vec3(fragColour.a), vec3(1.f,1.f,1.f), vec3(float(alphaTest)));
+		float fogFactorAdjust = mix(1.f, 0.f, float(alphaTest));
+		fragColour = vec4(mix(fragColour.rgb, fogColor.rgb * fogPremultAlphaFactor, clamp(fogFactor * fogFactorAdjust, 0.0, 1.0)), fragColour.a);
+		fragColour.a = fragColour.a * (1.0 - clamp(fogFactor, 0.0, 1.0));
 	}
 
 	#ifdef NEWGL
diff --git a/data/base/shaders/vk/nolight_instanced.frag b/data/base/shaders/vk/nolight_instanced.frag
index cd87f658259..3ae341d2b92 100644
--- a/data/base/shaders/vk/nolight_instanced.frag
+++ b/data/base/shaders/vk/nolight_instanced.frag
@@ -40,7 +40,10 @@ void main()
 		}
 
 		// Return fragment color
-		fragColour = mix(fragColour, vec4(fogColor.xyz, fragColour.w), clamp(fogFactor, 0.0, 1.0));
+		vec3 fogPremultAlphaFactor = mix(vec3(fragColour.a), vec3(1.f,1.f,1.f), vec3(float(alphaTest)));
+		float fogFactorAdjust = mix(1.f, 0.f, float(alphaTest));
+		fragColour = vec4(mix(fragColour.rgb, fogColor.rgb * fogPremultAlphaFactor, clamp(fogFactor * fogFactorAdjust, 0.0, 1.0)), fragColour.a);
+		fragColour.a = fragColour.a * (1.0 - clamp(fogFactor, 0.0, 1.0));
 	}
 
 	FragColor = fragColour;