Fixes for lightmap + new single-pass terrain renderer

Additive effects color (from things like landing zone lights, explosion effects, etc) was effectively not visible or lighting terrain when using the new single-pass renderer. This was a byproduct of how most lighting calculations have moved to the shader, and the lightmap value (and how additive effect lighting was accumulated) was being calculated differently.

This adjusts the lightmap to now always be 4 channels. When using the new single-pass shaders, the lightmap contains:
1. Additive color from effects (etc) in the .rgb channels
2. The calculated ambient occlusion value in the 4th (.a) channel

The shaders can then handle these two values separately.

(When using the old fallback terrain shaders, the first 3 channels contain the old calculated fully-inclusive-but-not-the-most-accurate illumination value. Only minor changes were required to these shaders to ignore the 4th channel.)

data/base/shaders/decals.frag

@@ -37,7 +37,7 @@ out vec4 FragColor;
 
 void main()
 {
-	vec4 fragColor = texture(tex, uv_tex, WZ_MIP_LOAD_BIAS) * texture(lightmap_tex, uv_lightmap, 0.f);
+	vec4 fragColor = texture(tex, uv_tex, WZ_MIP_LOAD_BIAS) * vec4(texture(lightmap_tex, uv_lightmap, 0.f).rgb, 1.f);
 
 	if (fogEnabled > 0)
 	{

data/base/shaders/terrain.frag

@@ -39,7 +39,7 @@ out vec4 FragColor;
 
 void main()
 {
-	vec4 fragColor = color * texture(tex, uv1, WZ_MIP_LOAD_BIAS) * texture(lightmap_tex, uv2, 0.f);
+	vec4 fragColor = color * texture(tex, uv1, WZ_MIP_LOAD_BIAS) * vec4(texture(lightmap_tex, uv2, 0.f).rgb, 1.f);
 
 	if (fogEnabled > 0)
 	{

data/base/shaders/terrain_combined_classic.frag

@@ -69,13 +69,19 @@ out vec4 FragColor;
 // Uses gl_FragColor
 #endif
 
+vec3 blendAddEffectLighting(vec3 a, vec3 b) {
+	return min(a + b, vec3(1.0));
+}
+
 vec4 main_classic() {
 	vec4 decal = tile >= 0 ? texture2DArray(decalTex, vec3(uvDecal, tile), WZ_MIP_LOAD_BIAS) : vec4(0.f);
 
 	vec3 L = normalize(groundLightDir);
 	vec3 N = vec3(0.f,0.f,1.f);
 	float lambertTerm = max(dot(N, L), 0.0); // diffuse lighting
-	vec4 light = (diffuseLight*0.75*lambertTerm + ambientLight*0.25) * texture(lightmap_tex, uvLightmap, 0.f);
+	vec4 lightmap_vec4 = texture(lightmap_tex, uvLightmap, 0.f);
+	vec4 light = (diffuseLight*0.75*lambertTerm + ambientLight*0.25) * lightmap_vec4.a; // ... * tile brightness / ambient occlusion (stored in lightmap.a);
+	light.rgb = blendAddEffectLighting(light.rgb, (lightmap_vec4.rgb / 1.5f)); // additive color (from environmental point lights / effects)
 	light.a = 1.f;
 
 	return light * decal;

data/base/shaders/terrain_combined_high.frag

@@ -111,6 +111,10 @@ vec4 doBumpMapping(BumpData b, vec3 lightDir, vec3 halfVec) {
 	return vec4(res.rgb, b.color.a);
 }
 
+vec3 blendAddEffectLighting(vec3 a, vec3 b) {
+	return min(a + b, vec3(1.0));
+}
+
 vec4 main_bumpMapping() {
 	BumpData bump;
 	bump.color = vec4(0.f);
@@ -137,8 +141,10 @@ vec4 main_bumpMapping() {
 		bump.N = (1.f - a)*bump.N + a*n;
 		bump.gloss = (1.f - a)*bump.gloss + a*texture2DArray(decalSpecular, uv, WZ_MIP_LOAD_BIAS).r;
 	}
-	vec4 lightMask = texture(lightmap_tex, uvLightmap, 0.f);
-	return lightMask * doBumpMapping(bump, groundLightDir, groundHalfVec);
+	vec4 lightmap_vec4 = texture(lightmap_tex, uvLightmap, 0.f);
+	vec4 color = doBumpMapping(bump, groundLightDir, groundHalfVec) * vec4(vec3(lightmap_vec4.a), 1.f); // ... * tile brightness / ambient occlusion (stored in lightmap.a);
+	color.rgb = blendAddEffectLighting(color.rgb, (lightmap_vec4.rgb / 1.5f)); // additive color (from environmental point lights / effects)
+	return color;
 }
 
 void main()

data/base/shaders/terrain_combined_medium.frag

@@ -78,14 +78,20 @@ vec3 getGround(int i) {
 	return texture2DArray(groundTex, getGroundUv(i), WZ_MIP_LOAD_BIAS).rgb * fgroundWeights[i];
 }
 
+vec3 blendAddEffectLighting(vec3 a, vec3 b) {
+	return min(a + b, vec3(1.0));
+}
+
 vec4 main_medium() {
 	vec3 ground = getGround(0) + getGround(1) + getGround(2) + getGround(3);
 	vec4 decal = tile >= 0 ? texture2DArray(decalTex, vec3(uvDecal, tile), WZ_MIP_LOAD_BIAS) : vec4(0.f);
 
 	vec3 L = normalize(groundLightDir);
 	vec3 N = vec3(0.f,0.f,1.f);
 	float lambertTerm = max(dot(N, L), 0.0); // diffuse lighting
-	vec4 light = (diffuseLight*0.75*lambertTerm + ambientLight*0.25) * texture(lightmap_tex, uvLightmap, 0.f);
+	vec4 lightmap_vec4 = texture(lightmap_tex, uvLightmap, 0.f);
+	vec4 light = (diffuseLight*0.75*lambertTerm + ambientLight*0.25) * lightmap_vec4.a; // ... * tile brightness / ambient occlusion (stored in lightmap.a)
+	light.rgb = blendAddEffectLighting(light.rgb, (lightmap_vec4.rgb / 1.5f)); // additive color (from environmental point lights / effects)
 	light.a = 1.f;
 
 	return light * vec4((1.f - decal.a) * ground + decal.a * decal.rgb, 1.f);

data/base/shaders/vk/decals.frag

@@ -24,7 +24,7 @@ layout(location = 0) out vec4 FragColor;
 
 void main()
 {
-	vec4 fragColor = texture(tex, uv_tex, WZ_MIP_LOAD_BIAS) * texture(lightmap_tex, uv_lightmap, 0.f);
+	vec4 fragColor = texture(tex, uv_tex, WZ_MIP_LOAD_BIAS) * vec4(texture(lightmap_tex, uv_lightmap, 0.f).rgb, 1.f);
 
 	if (fogEnabled > 0)
 	{

data/base/shaders/vk/terrain.frag

@@ -28,7 +28,7 @@ layout(location = 0) out vec4 FragColor;
 
 void main()
 {
-	vec4 fragColor = color * texture(tex, uv1, WZ_MIP_LOAD_BIAS) * texture(lightmap_tex, uv2, 0.f);
+	vec4 fragColor = color * texture(tex, uv1, WZ_MIP_LOAD_BIAS) * vec4(texture(lightmap_tex, uv2, 0.f).rgb, 1.f);
 
 	if (fogEnabled > 0)
 	{

data/base/shaders/vk/terrain_combined_classic.frag

@@ -23,13 +23,19 @@ layout(location = 9) flat in FragFlatData fragf;
 
 layout(location = 0) out vec4 FragColor;
 
+vec3 blendAddEffectLighting(vec3 a, vec3 b) {
+	return min(a + b, vec3(1.0));
+}
+
 vec4 main_classic() {
 	vec4 decal = fragf.tileNo >= 0 ? texture(decalTex, vec3(frag.uvDecal, fragf.tileNo), WZ_MIP_LOAD_BIAS) : vec4(0);
 
 	vec3 L = normalize(frag.groundLightDir);
 	vec3 N = vec3(0,0,1);
 	float lambertTerm = max(dot(N, L), 0.0); // diffuse lighting
-	vec4 light = (diffuseLight*0.75*lambertTerm + ambientLight*0.25) * texture(lightmap_tex, frag.uvLightmap);
+	vec4 lightmap_vec4 = texture(lightmap_tex, frag.uvLightmap);
+	vec4 light = (diffuseLight*0.75*lambertTerm + ambientLight*0.25) * lightmap_vec4.a; // ... * tile brightness / ambient occlusion (stored in lightmap.a);
+	light.rgb = blendAddEffectLighting(light.rgb, (lightmap_vec4.rgb / 1.5f)); // additive color (from environmental point lights / effects)
 	light.a = 1.f;
 
 	return light * decal;

data/base/shaders/vk/terrain_combined_high.frag

@@ -65,6 +65,10 @@ vec4 doBumpMapping(BumpData b, vec3 lightDir, vec3 halfVec) {
 	return vec4(res.rgb, b.color.a);
 }
 
+vec3 blendAddEffectLighting(vec3 a, vec3 b) {
+	return min(a + b, vec3(1.0));
+}
+
 vec4 main_bumpMapping() {
 	BumpData bump;
 	bump.color = vec4(0);
@@ -91,8 +95,10 @@ vec4 main_bumpMapping() {
 		bump.N = (1-a)*bump.N + a*n;
 		bump.gloss = (1-a)*bump.gloss + a*texture(decalSpecular, uv, WZ_MIP_LOAD_BIAS).r;
 	}
-	vec4 lightMask = texture(lightmap_tex, frag.uvLightmap);
-	return lightMask * doBumpMapping(bump, frag.groundLightDir, frag.groundHalfVec);
+	vec4 lightmap_vec4 = texture(lightmap_tex, frag.uvLightmap);
+	vec4 color = doBumpMapping(bump, frag.groundLightDir, frag.groundHalfVec) * vec4(vec3(lightmap_vec4.a), 1.f); // ... * tile brightness / ambient occlusion (stored in lightmap.a);
+	color.rgb = blendAddEffectLighting(color.rgb, (lightmap_vec4.rgb / 1.5f)); // additive color (from environmental point lights / effects)
+	return color;
 }
 
 void main()

data/base/shaders/vk/terrain_combined_medium.frag

@@ -32,14 +32,20 @@ vec3 getGround(int i) {
 	return texture(groundTex, getGroundUv(i), WZ_MIP_LOAD_BIAS).rgb * frag.groundWeights[i];
 }
 
+vec3 blendAddEffectLighting(vec3 a, vec3 b) {
+	return min(a + b, vec3(1.0));
+}
+
 vec4 main_medium() {
 	vec3 ground = getGround(0) + getGround(1) + getGround(2) + getGround(3);
 	vec4 decal = fragf.tileNo >= 0 ? texture(decalTex, vec3(frag.uvDecal, fragf.tileNo), WZ_MIP_LOAD_BIAS) : vec4(0.f);
 
 	vec3 L = normalize(frag.groundLightDir);
 	vec3 N = vec3(0.f,0.f,1.f);
 	float lambertTerm = max(dot(N, L), 0.0); // diffuse lighting
-	vec4 light = (diffuseLight*0.75*lambertTerm + ambientLight*0.25) * texture(lightmap_tex, frag.uvLightmap);
+	vec4 lightmap_vec4 = texture(lightmap_tex, frag.uvLightmap);
+	vec4 light = (diffuseLight*0.75*lambertTerm + ambientLight*0.25) * lightmap_vec4.a; // ... * tile brightness / ambient occlusion (stored in lightmap.a)
+	light.rgb = blendAddEffectLighting(light.rgb, (lightmap_vec4.rgb / 1.5f)); // additive color (from environmental point lights / effects)
 	light.a = 1.f;
 
 	return light * vec4((1.f - decal.a) * ground + decal.a * decal.rgb, 1.f);

src/display.cpp

@@ -2084,11 +2084,11 @@ void	dealWithLMB()
 			flipVal += 1;
 		}
 
-		console("%s tile %d, %d [%d, %d] continent(l%d, h%d) level %g illum %d ao %g col %x %s %s w=%d s=%d j=%d tile#%d (decal=%s, ground [#%d, size=%.3f], f%d r%d)",
+		console("%s tile %d, %d [%d, %d] continent(l%d, h%d) level %g illum %d ao %d col %x %s %s w=%d s=%d j=%d tile#%d (decal=%s, ground [#%d, size=%.3f], f%d r%d)",
 		        tileIsExplored(psTile) ? "Explored" : "Unexplored",
 		        mouseTileX, mouseTileY, world_coord(mouseTileX), world_coord(mouseTileY),
 		        (int)psTile->limitedContinent, (int)psTile->hoverContinent, psTile->level, (int)psTile->illumination,
-				psTile->ambientOcclusion, psTile->colour.rgba,
+				(int)psTile->ambientOcclusion, psTile->colour.rgba,
 		        aux & AUXBITS_DANGER ? "danger" : "", aux & AUXBITS_THREAT ? "threat" : "",
 		        (int)psTile->watchers[selectedPlayer], (int)psTile->sensors[selectedPlayer], (int)psTile->jammers[selectedPlayer],
 				TileNumber_tile(psTile->texture), (TILE_HAS_DECAL(psTile)) ? "y" : "n",

src/display3d.cpp

@@ -1328,6 +1328,7 @@ static void drawTiles(iView *player)
 
 	// update the fog of war... FIXME: Remove this
 	const glm::mat4 tileCalcPerspectiveViewMatrix = perspectiveMatrix * baseViewMatrix;
+	auto currTerrainShaderType = getTerrainShaderType();
 	for (int i = -visibleTiles.y / 2, idx = 0; i <= visibleTiles.y / 2; i++, ++idx)
 	{
 		/* Go through the x's */
@@ -1345,7 +1346,7 @@ static void drawTiles(iView *player)
 				MAPTILE *psTile = mapTile(playerXTile + j, playerZTile + i);
 
 				pos.y = map_TileHeight(playerXTile + j, playerZTile + i);
-				setTileColour(playerXTile + j, playerZTile + i, pal_SetBrightness(static_cast<UBYTE>(psTile->level)));
+				setTileColour(playerXTile + j, playerZTile + i, pal_SetBrightness((currTerrainShaderType == TerrainShaderType::SINGLE_PASS) ? 0 : static_cast<UBYTE>(psTile->level)));
 			}
 			tileScreenInfo[idx][jdx].z = pie_RotateProjectWithPerspective(&pos, tileCalcPerspectiveViewMatrix, &screen);
 			tileScreenInfo[idx][jdx].x = screen.x;

src/lighting.cpp

@@ -234,7 +234,7 @@ static void calcTileIllum(UDWORD tileX, UDWORD tileY)
 
 	MAPTILE *tile = mapTile(tileX, tileY);
 	tile->illumination = static_cast<uint8_t>(clip<int>(static_cast<int>(abs(dotProduct*ao)), 24, 254));
-	tile->ambientOcclusion = clip<float>(254.f*ao, 60.f, 254.f);
+	tile->ambientOcclusion = static_cast<uint8_t>(clip<float>(254.f*ao, 60.f, 254.f));
 }
 
 static void colourTile(SDWORD xIndex, SDWORD yIndex, PIELIGHT light_colour, double fraction)

src/map.h

@@ -74,7 +74,7 @@ struct MAPTILE
 	// DISPLAY ONLY (NOT for use in game calculations)
 	uint8_t         ground;                 ///< The ground type used for the terrain renderer
 	uint8_t         illumination;           // How bright is this tile? = diffuseSunLight * ambientOcclusion
-	float			ambientOcclusion;		// ambient occlusion. from 1 (max occlusion) to 254 (no occlusion), similar to illumination.
+	uint8_t			ambientOcclusion;		// ambient occlusion. from 1 (max occlusion) to 254 (no occlusion), similar to illumination.
 	float           level;                  ///< The visibility level of the top left of the tile, for this client. for terrain lightmap
 	PIELIGHT        colour;					// color in terrain lightmap, based on tile.level and near light sources
 };

src/terrain.cpp

@@ -1381,26 +1381,13 @@ bool initTerrain()
 	// Prepare the lightmap pixmap and texture
 	lightmapPixmap = std::unique_ptr<iV_Image>(new iV_Image());
 
-	// not every system may have RGB texture support, so may need to expand to 4-channel (RGBA)
-	auto lightmapChannels = gfx_api::context::get().getClosestSupportedUncompressedImageFormatChannels(gfx_api::pixel_format_target::texture_2d, 3); // ideal is RGB, but check if supported
-	ASSERT_OR_RETURN(false, lightmapChannels.has_value(), "Exhausted all possible uncompressed formats for lightmap texture??");
-
-	if (lightmapPixmap == nullptr || !lightmapPixmap->allocate(lightmapWidth, lightmapHeight, lightmapChannels.value(), true))
+	unsigned int lightmapChannels = 4; // always use 4-channel (RGBA)
+	if (lightmapPixmap == nullptr || !lightmapPixmap->allocate(lightmapWidth, lightmapHeight, lightmapChannels, true))
 	{
 		debug(LOG_FATAL, "Out of memory!");
 		abort();
 		return false;
 	}
-	if (lightmapPixmap->channels() == 4)
-	{
-		// must set alpha channel to opaque
-		auto lightmapWritePtr = lightmapPixmap->bmp_w();
-		size_t lightmapNumPixels = static_cast<size_t>(lightmapPixmap->width()) * static_cast<size_t>(lightmapPixmap->height());
-		for (size_t pixelIndex = 0; pixelIndex < lightmapNumPixels; ++pixelIndex)
-		{
-			lightmapWritePtr[pixelIndex * 4 + 3] = 255;
-		}
-	}
 	if (lightmap_texture)
 		delete lightmap_texture;
 	lightmap_texture = gfx_api::context::get().create_texture(1, lightmapPixmap->width(), lightmapPixmap->height(), lightmapPixmap->pixel_format(), "mem::lightmap");
@@ -1470,7 +1457,7 @@ void shutdownTerrain()
 
 static void updateLightMap()
 {
-	size_t lightmapChannels = lightmapPixmap->channels();
+	size_t lightmapChannels = lightmapPixmap->channels(); // should always be 4 now...
 	unsigned char* lightMapWritePtr = lightmapPixmap->bmp_w();
 	for (int j = 0; j < mapHeight; ++j)
 	{
@@ -1492,6 +1479,10 @@ static void updateLightMap()
 			lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 0] = colour.byte.r;
 			lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 1] = colour.byte.g;
 			lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 2] = colour.byte.b;
+			// store the "brightness" level in byte.a
+			// NOTE: This differs depending on whether using the single-pass terrain shader or the fallback terrain shaders
+			// (For more, see avUpdateTiles() and getTileIllumination())
+			lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 3] = static_cast<UBYTE>(psTile->level);
 
 			if (!pie_GetFogStatus())
 			{
@@ -1527,12 +1518,14 @@ static void updateLightMap()
 					lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 0] = 0;
 					lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 1] = 0;
 					lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 2] = 0;
+					lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 3] = 0;
 				}
 				else if (darken < 1)
 				{
 					lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 0] *= darken;
 					lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 1] *= darken;
 					lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 2] *= darken;
+					lightMapWritePtr[(i + j * lightmapWidth) * lightmapChannels + 3] *= darken;
 				}
 			}
 		}