From 77b96114c30232c1d3f3b548bf982f89f11d09da Mon Sep 17 00:00:00 2001
From: Ptolemy <ptolemy@lindenlab.com>
Date: Tue, 21 Jun 2022 22:00:12 -0700
Subject: [PATCH] SL-17274 Cleanup hard-coded gbuffer atmos flag for
 softenlight

---
 indra/llrender/llshadermgr.cpp                         |  7 ++++++-
 .../app_settings/shaders/class1/deferred/avatarF.glsl  |  2 +-
 .../app_settings/shaders/class1/deferred/bumpF.glsl    |  2 +-
 .../app_settings/shaders/class1/deferred/cloudsF.glsl  |  2 +-
 .../shaders/class1/deferred/diffuseAlphaMaskF.glsl     |  2 +-
 .../class1/deferred/diffuseAlphaMaskIndexedF.glsl      |  2 +-
 .../class1/deferred/diffuseAlphaMaskNoColorF.glsl      |  2 +-
 .../app_settings/shaders/class1/deferred/diffuseF.glsl |  2 +-
 .../shaders/class1/deferred/diffuseIndexedF.glsl       |  2 +-
 .../shaders/class1/deferred/highlightF.glsl            |  2 +-
 .../shaders/class1/deferred/impostorF.glsl             |  2 +-
 .../shaders/class1/deferred/materialF.glsl             | 10 +++++-----
 .../app_settings/shaders/class1/deferred/moonF.glsl    |  2 +-
 .../app_settings/shaders/class1/deferred/skyF.glsl     |  2 +-
 .../app_settings/shaders/class1/deferred/starsF.glsl   |  2 +-
 .../app_settings/shaders/class1/deferred/sunDiscF.glsl |  2 +-
 .../app_settings/shaders/class1/deferred/terrainF.glsl |  2 +-
 .../app_settings/shaders/class1/deferred/treeF.glsl    |  2 +-
 .../shaders/class1/deferred/underWaterF.glsl           |  2 +-
 .../app_settings/shaders/class1/deferred/waterF.glsl   |  2 +-
 .../app_settings/shaders/class2/deferred/skyF.glsl     |  2 +-
 .../shaders/class3/deferred/materialF.glsl             |  6 +++---
 22 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp
index e2e1ff9714e..f1996b65e96 100644
--- a/indra/llrender/llshadermgr.cpp
+++ b/indra/llrender/llshadermgr.cpp
@@ -787,7 +787,12 @@ GLhandleARB LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shade
 			extra_code_text[extra_code_count++] = strdup("#define shadow2DRect(a,b) vec2(texture(a,b))\n");
 		}
 	}
-	
+
+    // Use alpha float to store bit flags
+    // See: C++: addDeferredAttachment(), shader: frag_data[2]
+    extra_code_text[extra_code_count++] = strdup("#define GBUFFER_FLAG_SKIP_ATMOS   1.0\n"); // atmo kill
+    extra_code_text[extra_code_count++] = strdup("#define GBUFFER_FLAG_HAS_ATMOS    0.0\n");
+
 	if (defines)
 	{
 		for (std::unordered_map<std::string,std::string>::iterator iter = defines->begin(); iter != defines->end(); ++iter)
diff --git a/indra/newview/app_settings/shaders/class1/deferred/avatarF.glsl b/indra/newview/app_settings/shaders/class1/deferred/avatarF.glsl
index 60d83cc6232..22b3bd14634 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/avatarF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/avatarF.glsl
@@ -52,6 +52,6 @@ void main()
 	frag_data[0] = vec4(diff.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0);
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, 0.0);
+	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/bumpF.glsl b/indra/newview/app_settings/shaders/class1/deferred/bumpF.glsl
index b5677a07ee6..749ec3a6acc 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/bumpF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/bumpF.glsl
@@ -64,5 +64,5 @@ void main()
 		frag_data[1] = vertex_color.aaaa; // spec
 		//frag_data[1] = vec4(vec3(vertex_color.a), vertex_color.a+(1.0-vertex_color.a)*vertex_color.a); // spec - from former class3 - maybe better, but not so well tested
 		vec3 nvn = normalize(tnorm);
-		frag_data[2] = vec4(encode_normal(nvn), vertex_color.a, 0.0);	
+		frag_data[2] = vec4(encode_normal(nvn), vertex_color.a, GBUFFER_FLAG_HAS_ATMOS);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/cloudsF.glsl b/indra/newview/app_settings/shaders/class1/deferred/cloudsF.glsl
index ae1ac5de7f0..348e0f52289 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/cloudsF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/cloudsF.glsl
@@ -124,7 +124,7 @@ void main()
     /// Gamma correct for WL (soft clip effect).
     frag_data[0] = vec4(color.rgb, alpha1);
     frag_data[1] = vec4(0.0,0.0,0.0,0.0);
-    frag_data[2] = vec4(0,0,0,1);
+    frag_data[2] = vec4(0,0,0,GBUFFER_FLAG_SKIP_ATMOS);
 
     gl_FragDepth = 0.99995f;
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskF.glsl
index b328ee94832..3bf148502c0 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskF.glsl
@@ -53,6 +53,6 @@ void main()
 	frag_data[0] = vec4(col.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0); // spec
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, 0.0);
+	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskIndexedF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskIndexedF.glsl
index fc5c86b4d69..e15239b59d8 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskIndexedF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskIndexedF.glsl
@@ -52,5 +52,5 @@ void main()
 	frag_data[0] = vec4(col.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0);
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, 0.0);
+	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskNoColorF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskNoColorF.glsl
index 1bb8eb8bd05..b0ff2334149 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskNoColorF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskNoColorF.glsl
@@ -52,6 +52,6 @@ void main()
 	frag_data[0] = vec4(col.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0); // spec
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, 0.0);
+	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseF.glsl
index 8319e61242f..b2d2e2fa71f 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseF.glsl
@@ -46,6 +46,6 @@ void main()
 	frag_data[1] = vertex_color.aaaa; // spec
 	//frag_data[1] = vec4(vec3(vertex_color.a), vertex_color.a+(1.0-vertex_color.a)*vertex_color.a); // spec - from former class3 - maybe better, but not so well tested
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), vertex_color.a, 0.0);
+	frag_data[2] = vec4(encode_normal(nvn.xyz), vertex_color.a, GBUFFER_FLAG_HAS_ATMOS);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseIndexedF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseIndexedF.glsl
index ccd1df84f95..b4bc114dd55 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseIndexedF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseIndexedF.glsl
@@ -48,5 +48,5 @@ void main()
 	frag_data[0] = vec4(col, 0.0);
 	frag_data[1] = vec4(spec, vertex_color.a); // spec
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), vertex_color.a, 0.0);
+	frag_data[2] = vec4(encode_normal(nvn.xyz), vertex_color.a, GBUFFER_FLAG_HAS_ATMOS);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/highlightF.glsl b/indra/newview/app_settings/shaders/class1/deferred/highlightF.glsl
index 90566393d28..75f914cb029 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/highlightF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/highlightF.glsl
@@ -38,5 +38,5 @@ void main()
 {
 	frag_data[0] = color*texture2D(diffuseMap, vary_texcoord0.xy);
 	frag_data[1] = vec4(0.0);
-	frag_data[2] = vec4(0.0, 1.0, 0.0, 1.0);
+	frag_data[2] = vec4(0.0, 1.0, 0.0, GBUFFER_FLAG_SKIP_ATMOS);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl b/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
index a58cc3d12df..67f4c59c3f5 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
@@ -56,5 +56,5 @@ void main()
 
 	frag_data[0] = vec4(col.rgb, 0.0);
 	frag_data[1] = spec;
-	frag_data[2] = norm;
+	frag_data[2] = norm; // TODO: Should .w be set?
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/materialF.glsl b/indra/newview/app_settings/shaders/class1/deferred/materialF.glsl
index 02d83925eaa..6f087632a5d 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/materialF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/materialF.glsl
@@ -277,10 +277,10 @@ void main()
     vec4 final_specular = spec;
     
 #ifdef HAS_SPECULAR_MAP
-    vec4 final_normal = vec4(encode_normal(normalize(tnorm)), env_intensity * spec.a, 0.0);
+    vec4 final_normal = vec4(encode_normal(normalize(tnorm)), env_intensity * spec.a, GBUFFER_FLAG_HAS_ATMOS);
 	final_specular.a = specular_color.a * norm.a;
 #else
-	vec4 final_normal = vec4(encode_normal(normalize(tnorm)), env_intensity, 0.0);
+	vec4 final_normal = vec4(encode_normal(normalize(tnorm)), env_intensity, GBUFFER_FLAG_HAS_ATMOS);
 	final_specular.a = specular_color.a;
 #endif
 
@@ -444,10 +444,10 @@ void main()
 
 #else // mode is not DIFFUSE_ALPHA_MODE_BLEND, encode to gbuffer 
 
-    // deferred path
-    frag_data[0] = final_color; //gbuffer is sRGB
+    // deferred path               // See: C++: addDeferredAttachment(), shader: softenLightF.glsl
+    frag_data[0] = final_color;    // gbuffer is sRGB
     frag_data[1] = final_specular; // XYZ = Specular color. W = Specular exponent.
-    frag_data[2] = final_normal; // XY = Normal.  Z = Env. intensity.
+    frag_data[2] = final_normal;   // XY = Normal.  Z = Env. intensity. W = 1 skip atmos (mask off fog)
 #endif
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/moonF.glsl b/indra/newview/app_settings/shaders/class1/deferred/moonF.glsl
index 35068899eee..a513d60388f 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/moonF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/moonF.glsl
@@ -66,7 +66,7 @@ void main()
 
     frag_data[0] = vec4(c.rgb, c.a);
     frag_data[1] = vec4(0.0);
-    frag_data[2] = vec4(0.0f);
+    frag_data[2] = vec4(0.0, 0.0, 0.0, GBUFFER_FLAG_HAS_ATMOS);
 
     gl_FragDepth = 0.999985f;
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/skyF.glsl b/indra/newview/app_settings/shaders/class1/deferred/skyF.glsl
index 331249dc33b..ecb0c43518a 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/skyF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/skyF.glsl
@@ -57,7 +57,7 @@ void main()
     /// Gamma correct for WL (soft clip effect).
     frag_data[0] = vec4(color.rgb, 0.0);
     frag_data[1] = vec4(0.0,0.0,0.0,0.0);
-    frag_data[2] = vec4(0.0,0.0,0.0,1.0); //1.0 in norm.w masks off fog
+    frag_data[2] = vec4(0.0,0.0,0.0,GBUFFER_FLAG_SKIP_ATMOS); //1.0 in norm.w masks off fog
 
     gl_FragDepth = 0.99999f;
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/starsF.glsl b/indra/newview/app_settings/shaders/class1/deferred/starsF.glsl
index bac79a9fdc7..ef9cee3fa06 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/starsF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/starsF.glsl
@@ -60,7 +60,7 @@ void main()
 
     frag_data[0] = col;
     frag_data[1] = vec4(0.0f);
-    frag_data[2] = vec4(0.0, 1.0, 0.0, 1.0);
+    frag_data[2] = vec4(0.0, 1.0, 0.0, GBUFFER_FLAG_SKIP_ATMOS);
 
     gl_FragDepth = 0.99995f;
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/sunDiscF.glsl b/indra/newview/app_settings/shaders/class1/deferred/sunDiscF.glsl
index b2fa5d8a256..4ab8747629f 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/sunDiscF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/sunDiscF.glsl
@@ -57,7 +57,7 @@ void main()
 
     frag_data[0] = c;
     frag_data[1] = vec4(0.0f);
-    frag_data[2] = vec4(0.0, 1.0, 0.0, 1.0);
+    frag_data[2] = vec4(0.0, 1.0, 0.0, GBUFFER_FLAG_SKIP_ATMOS);
 
     gl_FragDepth = 0.999988f;
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/terrainF.glsl b/indra/newview/app_settings/shaders/class1/deferred/terrainF.glsl
index 6b6eed9db88..d6c14c48c93 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/terrainF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/terrainF.glsl
@@ -63,6 +63,6 @@ void main()
     frag_data[0] = outColor;
     frag_data[1] = vec4(0.0,0.0,0.0,-1.0);
     vec3 nvn = normalize(vary_normal);
-    frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, 0.0);
+    frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/treeF.glsl b/indra/newview/app_settings/shaders/class1/deferred/treeF.glsl
index 89e354558a2..dc0e5b0ce3c 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/treeF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/treeF.glsl
@@ -52,5 +52,5 @@ void main()
 	frag_data[0] = vec4(vertex_color.rgb*col.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0);
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, 0.0);
+	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/underWaterF.glsl b/indra/newview/app_settings/shaders/class1/deferred/underWaterF.glsl
index 9a5debb3c12..14c337e6087 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/underWaterF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/underWaterF.glsl
@@ -78,5 +78,5 @@ void main()
 
 	frag_data[0] = vec4(fb.rgb, 1.0); // diffuse
 	frag_data[1] = vec4(0.5,0.5,0.5, 0.95); // speccolor*spec, spec
-	frag_data[2] = vec4(encode_normal(wavef), 0.0, 0.0); // normalxyz, env intens, atmo kill
+	frag_data[2] = vec4(encode_normal(wavef), 0.0, GBUFFER_FLAG_HAS_ATMOS); // normalxyz, env intens, flags (atmo kill)
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/waterF.glsl b/indra/newview/app_settings/shaders/class1/deferred/waterF.glsl
index a157e9c017b..03896f4c51b 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/waterF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/waterF.glsl
@@ -180,5 +180,5 @@ void main()
 
 
     frag_data[1] = vec4(0);		// speccolor, spec
-	frag_data[2] = vec4(encode_normal(screenspacewavef.xyz), 0.05, 0);// normalxy, 0, 0
+	frag_data[2] = vec4(encode_normal(screenspacewavef.xyz), 0.05, GBUFFER_FLAG_HAS_ATMOS);// normalxy, env intens, flags (atmo kill)
 }
diff --git a/indra/newview/app_settings/shaders/class2/deferred/skyF.glsl b/indra/newview/app_settings/shaders/class2/deferred/skyF.glsl
index 6841a8194f6..4379024680b 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/skyF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/skyF.glsl
@@ -195,5 +195,5 @@ void main()
     // Gamma correct for WL (soft clip effect).
     frag_data[0] = vec4(color.rgb, 1.0);
     frag_data[1] = vec4(0.0, 0.0, 0.0, 0.0);
-    frag_data[2] = vec4(0.0, 0.0, 0.0, 1.0);  // 1.0 in norm.w masks off fog
+    frag_data[2] = vec4(0.0, 0.0, 0.0, GBUFFER_FLAG_SKIP_ATMOS);  // 1.0 in norm.w masks off fog
 }
diff --git a/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl b/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
index c5b1937cfb6..13369d09fd6 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
@@ -446,10 +446,10 @@ void main()
 
 #else // mode is not DIFFUSE_ALPHA_MODE_BLEND, encode to gbuffer 
 
-    // deferred path
-    frag_data[0] = final_color; //gbuffer is sRGB
+    // deferred path               // See: C++: addDeferredAttachment(), shader: softenLightF.glsl
+    frag_data[0] = final_color;    // gbuffer is sRGB
     frag_data[1] = final_specular; // XYZ = Specular color. W = Specular exponent.
-    frag_data[2] = final_normal; // XY = Normal.  Z = Env. intensity.
+    frag_data[2] = final_normal;   // XY = Normal.  Z = Env. intensity. W = 1 skip atmos (mask off fog)
 #endif
 }
 
-- 
GitLab