SL-15031 - more viewerstats.py support for cleaning up unused settings

scripts/metrics/viewerstats.py

@@ -34,6 +34,8 @@
 from llbase import llsd
 import io
 import re
+import os
+import sys
 
 def show_stats_by_key(recs,indices,settings_sd = None):
     result = ()
@@ -102,7 +104,7 @@ def parse_settings_xml(fname):
         contents = f.read()
         return llsd.parse_xml(contents)
 
-def read_settings_xml(fname):
+def read_raw_settings_xml(fname):
     # assume we're in scripts/metrics
     fname = "../../indra/newview/app_settings/" + fname
     contents = None
@@ -126,10 +128,38 @@ def write_raw_settings_xml(fname, string):
 
 def remove_settings(string, to_remove):
     for r in to_remove:
-        subs_str = r"<key>" + r + r"<.*?</map>"
+        subs_str = r"<key>" + r + r"<.*?</map>\n"
         string = re.sub(subs_str,"",string,flags=re.S|re.DOTALL)
     return string
 
+def get_used_strings(root_dir):
+    used_str = set()
+    skipped_ext = set()
+    for dir_name, sub_dir_list, file_list in os.walk(root_dir):
+        for fname in file_list:
+            if fname in ["settings.xml", "settings.xml.edit", "settings_per_account.xml"]:
+                print "skip", fname
+                continue
+            (base,ext) = os.path.splitext(fname)
+            #if ext not in [".cpp", ".hpp", ".h", ".xml"]:
+            #    skipped_ext.add(ext)
+            #    continue
+            
+            full_name = os.path.join(dir_name,fname)
+
+            with open(full_name,"r") as f:
+                #print full_name
+                lines = f.readlines()
+                for l in lines:
+                    ms = re.findall(r'[>\"]([A-Za-z0-9_]+)[\"<]',l)
+                    for m in ms:
+                        #print "used_str",m
+                        used_str.add(m)
+    print "skipped extensions", skipped_ext
+    print "got used_str", len(used_str)
+    return used_str
+                
+    
 if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description="process tab-separated table containing viewerstats logs")
@@ -159,24 +189,35 @@ def remove_settings(string, to_remove):
             (all_str,_,_,_) = show_stats_by_key(recs,["preferences","settings"],settings_sd)
             print
 
-            print "\nSETTINGS_PER_ACCOUNT.XML"
-            settings_pa_sd = parse_settings_xml("settings_per_account.xml")
-            show_stats_by_key(recs,["preferences","settings_per_account"],settings_pa_sd)
+            #print "\nSETTINGS_PER_ACCOUNT.XML"
+            #settings_pa_sd = parse_settings_xml("settings_per_account.xml")
+            #show_stats_by_key(recs,["preferences","settings_per_account"],settings_pa_sd)
 
             if args.remove_unused:
-                # quotestrings created by
-                # % find . -name '*.cpp' -o -name '*.hpp' -o -name '*.h' -o -name '*cmd_line.xml'  | xargs grep -ohP '[\">][a-zA-Z0-9]*[\"<]' | sort | uniq > ../scripts/metrics/quotestrings.out
-                with open("quotestrings.out", "r") as f:
-                    used_strings = f.readlines()
-                    used_strings = [u[1:-2] for u in used_strings]
-                    #print "\n".join(used_strings)
-                    unref_strings = list(set(all_str)-set(used_strings))
-                    print "\nUNREF_IN_CODE " + str(len(unref_strings)) + "\n"
-                    print "\n".join(unref_strings)
-                    settings_str = read_settings_xml("settings.xml")
-                    # Do this via direct string munging to generate minimal changeset
-                    settings_edited = remove_settings(settings_str,unref_strings)
-                    write_raw_settings_xml("settings.xml",settings_edited)
+                # walk codebase looking for strings
+                all_str_set = set(all_str)
+                used_strings = get_used_strings("../../indra")
+                used_strings_set = set(used_strings)
+                unref_strings = all_str_set-used_strings_set
+                # Some settings names are generated by appending to a prefix. Need to look for this case.
+                prefix_used = set()
+                print "checking unref_strings", len(unref_strings)
+                for u in unref_strings:
+                    for k in range(6,len(u)):
+                        prefix = u[0:k]
+                        if prefix in all_str_set and prefix in used_strings_set:
+                            prefix_used.add(u)
+                            #print "PREFIX_USED",u,prefix
+                print "PREFIX_USED", len(prefix_used), ",".join(list(prefix_used))
+                print
+                unref_strings = unref_strings - prefix_used
+                        
+                print "\nUNREF_IN_CODE " + str(len(unref_strings)) + "\n"
+                print "\n".join(list(unref_strings))
+                settings_str = read_raw_settings_xml("settings.xml")
+                # Do this via direct string munging to generate minimal changeset
+                settings_edited = remove_settings(settings_str,unref_strings)
+                write_raw_settings_xml("settings.xml.edit",settings_edited)