From 6bd18ea8a1494dd6e34de0be256a1e59cc2515eb Mon Sep 17 00:00:00 2001
From: Rye Mutt <rye@alchemyviewer.org>
Date: Wed, 4 Jan 2023 16:38:31 -0500
Subject: [PATCH] Fix SSAO-ish-kinda

---
 indra/llrender/llshadermgr.cpp                |  6 +-
 indra/llrender/llshadermgr.h                  |  6 +-
 indra/newview/app_settings/settings.xml       | 18 ++---
 .../shaders/class1/deferred/aoUtil.glsl       | 81 +++++++++----------
 .../shaders/class1/deferred/blurLightF.glsl   | 17 ++--
 .../class1/windlight/atmosphericsFuncs.glsl   | 22 +++--
 indra/newview/pipeline.cpp                    | 14 +++-
 7 files changed, 93 insertions(+), 71 deletions(-)

diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp
index b762c0bd7fd..a5f7f5f9e7e 100644
--- a/indra/llrender/llshadermgr.cpp
+++ b/indra/llrender/llshadermgr.cpp
@@ -1201,10 +1201,14 @@ void LLShaderMgr::initAttribsAndUniforms()
 	mReservedUniforms.push_back("shadow_matrix");
 	mReservedUniforms.push_back("env_mat");
 	mReservedUniforms.push_back("shadow_clip");
+	mReservedUniforms.push_back("sun_wash");
+	mReservedUniforms.push_back("shadow_noise");
+	mReservedUniforms.push_back("blur_size");
 	mReservedUniforms.push_back("ssao_radius");
 	mReservedUniforms.push_back("ssao_max_radius");
 	mReservedUniforms.push_back("ssao_factor");
-	mReservedUniforms.push_back("ssao_effect");
+	mReservedUniforms.push_back("ssao_factor_inv");
+	mReservedUniforms.push_back("ssao_effect_mat");
 	mReservedUniforms.push_back("screen_res");
 	mReservedUniforms.push_back("near_clip");
 	mReservedUniforms.push_back("shadow_offset");
diff --git a/indra/llrender/llshadermgr.h b/indra/llrender/llshadermgr.h
index 489a179663e..c7ef0064e8e 100644
--- a/indra/llrender/llshadermgr.h
+++ b/indra/llrender/llshadermgr.h
@@ -118,10 +118,14 @@ class LLShaderMgr
         DEFERRED_SHADOW_MATRIX,             //  "shadow_matrix"
         DEFERRED_ENV_MAT,                   //  "env_mat"
         DEFERRED_SHADOW_CLIP,               //  "shadow_clip"
+		DEFERRED_SUN_WASH,
+		DEFERRED_SHADOW_NOISE,
+		DEFERRED_BLUR_SIZE,
         DEFERRED_SSAO_RADIUS,               //  "ssao_radius"
         DEFERRED_SSAO_MAX_RADIUS,           //  "ssao_max_radius"
         DEFERRED_SSAO_FACTOR,               //  "ssao_factor"
-		DEFERRED_SSAO_EFFECT,               //  "ssao_effect"
+		DEFERRED_SSAO_FACTOR_INV,
+		DEFERRED_SSAO_EFFECT_MAT,
         DEFERRED_SCREEN_RES,                //  "screen_res"
         DEFERRED_NEAR_CLIP,                 //  "near_clip"
         DEFERRED_SHADOW_OFFSET,             //  "shadow_offset"
diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index a4136aeb10e..3779b9eb2e1 100644
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -9527,7 +9527,7 @@
     <key>Type</key>
     <string>F32</string>
     <key>Value</key>
-    <real>1500.0</real>
+    <real>500.0</real>
   </map>
   <key>RenderSSAOMaxScale</key>
   <map>
@@ -9538,7 +9538,7 @@
     <key>Type</key>
     <string>U32</string>
     <key>Value</key>
-    <integer>400</integer>
+    <integer>200</integer>
   </map>
   <key>RenderSSAOFactor</key>
   <map>
@@ -9549,7 +9549,7 @@
     <key>Type</key>
     <string>F32</string>
     <key>Value</key>
-    <real>5.0</real>
+    <real>0.30</real>
   </map>
   <key>RenderSSAOEffect</key>
   <map>
@@ -9561,7 +9561,7 @@
     <string>Vector3</string>
     <key>Value</key>
     <array>
-      <real>0.50</real>
+      <real>0.80</real>
       <real>1.00</real>
       <real>0.00</real>
     </array>
@@ -9802,7 +9802,7 @@
     <key>Type</key>
     <string>F32</string>
     <key>Value</key>
-    <real>-0.003</real>
+    <real>-0.004</real>
   </map>
   <key>RenderShadowOffset</key>
   <map>
@@ -9907,7 +9907,7 @@
     <key>Type</key>
     <string>F32</string>
     <key>Value</key>
-    <real>0.0</real>
+    <real>-0.001</real>
   </map>
   <key>RenderSpotShadowOffset</key>
   <map>
@@ -9918,7 +9918,7 @@
     <key>Type</key>
     <string>F32</string>
     <key>Value</key>
-    <real>0.15</real>
+    <real>0.04</real>
   </map>
 
   <key>RenderShadowResolutionScale</key>
@@ -10183,7 +10183,7 @@
     <string>Vector3</string>
     <key>Value</key>
     <array>
-      <real>1.0</real>
+      <real>3.0</real>
       <real>2.0</real>
       <real>0.0</real>
     </array>
@@ -10198,7 +10198,7 @@
     <key>Type</key>
     <string>F32</string>
     <key>Value</key>
-    <real>3.0</real>
+    <real>1.4</real>
   </map>
   <key>RenderShadowBlurSamples</key>
   <map>
diff --git a/indra/newview/app_settings/shaders/class1/deferred/aoUtil.glsl b/indra/newview/app_settings/shaders/class1/deferred/aoUtil.glsl
index c91716ce218..23adbded5e1 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/aoUtil.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/aoUtil.glsl
@@ -30,6 +30,7 @@ uniform sampler2DRect   depthMap;
 uniform float ssao_radius;
 uniform float ssao_max_radius;
 uniform float ssao_factor;
+uniform float ssao_factor_inv;
 
 uniform mat4 inv_proj;
 uniform vec2 screen_res;
@@ -80,49 +81,43 @@ vec2 getKern(int i)
 //calculate decreases in ambient lighting when crowded out (SSAO)
 float calcAmbientOcclusion(vec4 pos, vec3 norm, vec2 pos_screen)
 {
-  vec2 noise_reflect = texture2D(noiseMap, pos_screen.xy/128.0).xy;
-
-  // We treat the first sample as the origin, which definitely doesn't obscure itself thanks to being visible for sampling in the first place.
-  float points = 1.0;
-  float angle_hidden = 0.0;
-		
-  // use a kernel scale that diminishes with distance.
-  // a scale of less than 32 is just wasting good samples, though.
-  float scale = max(32.0, min(ssao_radius / -pos.z, ssao_max_radius));
-
-  // it was found that keeping # of samples a constant was the fastest, probably due to compiler optimizations (unrolling?)
-  for (int i = 0; i < 8; i++)
-  {
-    vec2 samppos_screen = pos_screen + scale * reflect(getKern(i), noise_reflect);
-
-    // if sample is out-of-screen then give it no weight by continuing
-    if (any(lessThan(samppos_screen.xy, vec2(0.0, 0.0))) ||
-	any(greaterThan(samppos_screen.xy, vec2(screen_res.xy)))) continue;
-
-    vec3 samppos_world = getPositionAo(samppos_screen).xyz; 
-			
-    vec3 diff = samppos_world - pos.xyz;
-
-    if (diff.z < ssao_factor // only use sample if it's near enough
-	&& diff.z != 0.0     // Z is very quantized at distance, this lessens noise and eliminates dist==0
-	)
+    float ret = 1.0;
+    vec3 pos_world = pos.xyz;
+    vec2 noise_reflect = texture2D(noiseMap, pos_screen.xy/128.0).xy;
+        
+    float angle_hidden = 0.0;
+    float points = 0;
+        
+    float scale = min(ssao_radius / -pos_world.z, ssao_max_radius);
+    
+    // it was found that keeping # of samples a constant was the fastest, probably due to compiler optimizations (unrolling?)
+    for (int i = 0; i < 8; i++)
     {
-	float dist = length(diff);
-	float angrel = max(0.0, dot(norm.xyz, diff/dist)); // how much the origin faces the sample
-	float distrel = 1.0/(1.0+dist*dist); // 'closeness' of origin to sample
-
-	// origin is obscured by this sample according to how directly the origin is facing the sample and how close the sample is.  It has to score high on both to be a good occluder.  (a*d) seems the most intuitive way to score, but min(a,d) gives a less localized effect...
-	float samplehidden = min(angrel, distrel);
-
-	angle_hidden += (samplehidden);
-	points += 1.0;
-      }
-  }
-
-  angle_hidden = angle_hidden / points;
-		
-  float rtn = (1.0 - angle_hidden);
-
-  return (rtn * rtn);
+        vec2 samppos_screen = pos_screen + scale * reflect(getKern(i), noise_reflect);
+        vec3 samppos_world = getPositionAo(samppos_screen).xyz; 
+        
+        vec3 diff = pos_world - samppos_world;
+        float dist2 = dot(diff, diff);
+            
+        // assume each sample corresponds to an occluding sphere with constant radius, constant x-sectional area
+        // --> solid angle shrinking by the square of distance
+        //radius is somewhat arbitrary, can approx with just some constant k * 1 / dist^2
+        //(k should vary inversely with # of samples, but this is taken care of later)
+        
+        float funky_val = (dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0) ? 1.0 : 0.0;
+        angle_hidden = angle_hidden + funky_val * min(1.0/dist2, ssao_factor_inv);
+            
+        // 'blocked' samples (significantly closer to camera relative to pos_world) are "no data", not "no occlusion" 
+        float diffz_val = (diff.z > -1.0) ? 1.0 : 0.0;
+        points = points + diffz_val;
+    }
+        
+    angle_hidden = min(ssao_factor*angle_hidden/points, 1.0);
+    
+    float points_val = (points > 0.0) ? 1.0 : 0.0;
+    ret = (1.0 - (points_val * angle_hidden));
+
+    ret = max(ret, 0.0);
+    return min(ret, 1.0);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/blurLightF.glsl b/indra/newview/app_settings/shaders/class1/deferred/blurLightF.glsl
index a9c45678c3a..596d0274af3 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/blurLightF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/blurLightF.glsl
@@ -33,9 +33,11 @@ out vec4 frag_color;
 #define frag_color gl_FragColor
 #endif
 
+uniform sampler2DRect normalMap;
 uniform sampler2DRect lightMap;
 
 uniform float dist_factor;
+uniform float blur_size;
 uniform vec2 delta;
 uniform vec3 kern[4];
 uniform float kern_scale;
@@ -52,23 +54,24 @@ void main()
     vec3 pos = getPosition(tc).xyz;
     vec4 ccol = texture2DRect(lightMap, tc).rgba;
     
-    vec2 dlt = kern_scale * delta / (vec2(1.0)+norm.xy*norm.xy);
+    vec2 dlt = kern_scale * delta / (1.0+norm.xy*norm.xy);
     dlt /= max(-pos.z*dist_factor, 1.0);
     
     vec2 defined_weight = kern[0].xy; // special case the first (centre) sample's weight in the blur; we have to sample it anyway so we get it for 'free'
     vec4 col = defined_weight.xyxx * ccol;
 
     // relax tolerance according to distance to avoid speckling artifacts, as angles and distances are a lot more abrupt within a small screen area at larger distances
-    float pointplanedist_tolerance_pow2 = pos.z*-0.001;
+    float pointplanedist_tolerance_pow2 = pos.z*pos.z*0.00005;
 
     // perturb sampling origin slightly in screen-space to hide edge-ghosting artifacts where smoothing radius is quite large
-    vec2 tc_v = fract(0.5 * tc.xy); // we now have floor(mod(tc,2.0))*0.5
-    float tc_mod = 2.0 * abs(tc_v.x - tc_v.y); // diff of x,y makes checkerboard
+    float tc_mod = 0.5*(tc.x + tc.y); // mod(tc.x+tc.y,2)
+    tc_mod -= floor(tc_mod);
+    tc_mod *= 2.0;
     tc += ( (tc_mod - 0.5) * kern[1].z * dlt * 0.5 );
 
-    for (int i = 3; i > 0; i--)
+    for (int i = 1; i < 4; i++)
     {
-        vec2 samptc = (tc + kern[i].z * dlt);
+        vec2 samptc = tc + kern[i].z*dlt;
         vec3 samppos = getPosition(samptc).xyz; 
 
         float d = dot(norm.xyz, samppos.xyz-pos.xyz);// dist from plane
@@ -82,7 +85,7 @@ void main()
 
     for (int i = 1; i < 4; i++)
     {
-        vec2 samptc = (tc - kern[i].z * dlt);
+        vec2 samptc = tc - kern[i].z*dlt;
         vec3 samppos = getPosition(samptc).xyz; 
 
         float d = dot(norm.xyz, samppos.xyz-pos.xyz);// dist from plane
diff --git a/indra/newview/app_settings/shaders/class1/windlight/atmosphericsFuncs.glsl b/indra/newview/app_settings/shaders/class1/windlight/atmosphericsFuncs.glsl
index 8d7327fd2f0..ea2690ba09d 100644
--- a/indra/newview/app_settings/shaders/class1/windlight/atmosphericsFuncs.glsl
+++ b/indra/newview/app_settings/shaders/class1/windlight/atmosphericsFuncs.glsl
@@ -37,7 +37,7 @@ uniform float distance_multiplier;
 uniform float max_y;
 uniform vec4  glow;
 uniform float scene_light_strength;
-uniform float  ssao_effect;
+uniform mat3  ssao_effect_mat;
 uniform int   no_atmo;
 uniform float sun_moon_glow_factor;
 
@@ -106,6 +106,20 @@ void calcAtmosphericVars(vec3 inPositionEye, vec3 light_dir, float ambFactor, ou
     // increase ambient when there are more clouds
     vec4 tmpAmbient = amb_color + (vec4(1.) - amb_color) * cloud_shadow * 0.5;
 
+    /*  decrease value and saturation (that in HSV, not HSL) for occluded areas
+     * // for HSV color/geometry used here, see http://gimp-savvy.com/BOOK/index.html?node52.html
+     * // The following line of code performs the equivalent of:
+     * float ambAlpha = tmpAmbient.a;
+     * float ambValue = dot(vec3(tmpAmbient), vec3(0.577)); // projection onto <1/rt(3), 1/rt(3), 1/rt(3)>, the neutral white-black axis
+     * vec3 ambHueSat = vec3(tmpAmbient) - vec3(ambValue);
+     * tmpAmbient = vec4(RenderSSAOEffect.valueFactor * vec3(ambValue) + RenderSSAOEffect.saturationFactor *(1.0 - ambFactor) * ambHueSat,
+     * ambAlpha);
+     */
+    if (use_ao)
+    {
+        tmpAmbient = vec4(mix(ssao_effect_mat * tmpAmbient.rgb, tmpAmbient.rgb, ambFactor), tmpAmbient.a);
+    }
+
     // Similar/Shared Algorithms:
     //     indra\llinventory\llsettingssky.cpp                                        -- LLSettingsSky::calculateLightSettings()
     //     indra\newview\app_settings\shaders\class1\windlight\atmosphericsFuncs.glsl -- calcAtmosphericVars()
@@ -113,12 +127,6 @@ void calcAtmosphericVars(vec3 inPositionEye, vec3 light_dir, float ambFactor, ou
     vec3 cs = sunlight.rgb * (1. - cloud_shadow);
     additive = (blue_horizon.rgb * blue_weight.rgb) * (cs + tmpAmbient.rgb) + (haze_horizon * haze_weight.rgb) * (cs * haze_glow + tmpAmbient.rgb);
 
-    // decrease ambient value for occluded areas
-    if (use_ao)
-    {
-        tmpAmbient *= mix(ssao_effect, 1.0, ambFactor);
-    }
-
     // brightness of surface both sunlight and ambient
     sunlit = sunlight.rgb * 0.5;
     amblit = tmpAmbient.rgb * .25;
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 034381fb7eb..3c9e325f70e 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -8595,10 +8595,18 @@ void LLPipeline::bindDeferredShader(LLGLSLShader& shader, LLRenderTarget* light_
 
 		F32 ssao_factor = RenderSSAOFactor;
 		shader.uniform1f(LLShaderMgr::DEFERRED_SSAO_FACTOR, ssao_factor);
+		shader.uniform1f(LLShaderMgr::DEFERRED_SSAO_FACTOR_INV, 1.f/ssao_factor);
 	}
 
 	LLVector3 ssao_effect = RenderSSAOEffect;
-	shader.uniform1f(LLShaderMgr::DEFERRED_SSAO_EFFECT, ssao_effect[0]);
+	F32 matrix_diag = (ssao_effect[0] + 2.0*ssao_effect[1])/3.0;
+	F32 matrix_nondiag = (ssao_effect[0] - ssao_effect[1])/3.0;
+	// This matrix scales (proj of color onto <1/rt(3),1/rt(3),1/rt(3)>) by
+	// value factor, and scales remainder by saturation factor
+	F32 ssao_effect_mat[] = {	matrix_diag, matrix_nondiag, matrix_nondiag,
+								matrix_nondiag, matrix_diag, matrix_nondiag,
+								matrix_nondiag, matrix_nondiag, matrix_diag};
+	shader.uniformMatrix3fv(LLShaderMgr::DEFERRED_SSAO_EFFECT_MAT, 1, GL_FALSE, ssao_effect_mat);
 
     shader.uniform2f(LLShaderMgr::DEFERRED_SCREEN_RES, deferred_target->getWidth(), deferred_target->getHeight());
 
@@ -8748,14 +8756,14 @@ void LLPipeline::renderDeferredLighting(LLRenderTarget *screen_target)
             bindDeferredShader(gDeferredBlurLightProgram);
             mDeferredVB->setBuffer(LLVertexBuffer::MAP_VERTEX);
             LLVector3 go          = RenderShadowGaussian;
-            const U32 kern_length = 4;
+            constexpr U32 kern_length = 4;
             F32       blur_size   = RenderShadowBlurSize;
             F32       dist_factor = RenderShadowBlurDistFactor;
 
             // sample symmetrically with the middle sample falling exactly on 0.0
             F32 x = 0.f;
 
-            LLVector3 gauss[4];  // xweight, yweight, offset
+            LLVector3 gauss[kern_length];  // xweight, yweight, offset
 
             for (U32 i = 0; i < kern_length; i++)
             {
-- 
GitLab