Better loading of static assets

indra/llcharacter/llkeyframemotion.cpp

@@ -301,48 +301,15 @@ LLMotion::LLMotionInitStatus LLKeyframeMotion::onInitialize(LLCharacter *charact
 		return STATUS_SUCCESS;
 	}
 
+	//-------------------------------------------------------------------------
+	// Load named file by concatenating the character prefix with the motion name.
+	// Load data into a buffer to be parsed.
+	//-------------------------------------------------------------------------
 	BOOL success = FALSE;
 	U8* anim_data = nullptr;
 	S32 anim_file_size = 0;
 
-	//-------------------------------------------------------------------------
-	// First attempt to find the animation in the static animation cache.
-	//-------------------------------------------------------------------------
-	std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_CHARACTER,
-		"animmations", gDirUtilp->getDirDelimiter(), mID.asString() + ".animatn");
-	LLUniqueFile fp = LLFile::fopen(filename, "rb");
-	if (fp)	// If the file exists
 	{
-		fseek(fp, 0L, SEEK_END);
-		anim_file_size = ftell(fp);
-		fseek(fp, 0L, SEEK_SET);
-
-		anim_data = new (std::nothrow) U8[anim_file_size];
-		if (anim_data)
-		{
-			success = fread(anim_data, 1,
-				anim_file_size, fp) == anim_file_size;
-			if (!success)
-			{
-				delete[] anim_data;
-				anim_data = nullptr;
-			}
-		}
-		else
-		{
-			LL_WARNS() << "Failed to allocate buffer: " << anim_file_size << mID << LL_ENDL;
-		}
-		fp.close();
-	}
-
-	if (!success)
-	{
-		//-------------------------------------------------------------------------
-		// Load named file by concatenating the character prefix with the motion name.
-		// Load data into a buffer to be parsed.
-		//-------------------------------------------------------------------------
-
-
 		LLFileSystem anim_file(mID, LLAssetType::AT_ANIMATION);
 		if (!anim_file.open() || !anim_file.getSize())
 		{
@@ -370,10 +337,6 @@ LLMotion::LLMotionInitStatus LLKeyframeMotion::onInitialize(LLCharacter *charact
 			}
 		}
 	}
-	else
-	{
-		LL_DEBUGS() << "Loaded keyframe data from static anim cache: " << mID << LL_ENDL;
-	}
 
 	if (!success)
 	{

indra/llfilesystem/lldiskcache.cpp

@@ -61,6 +61,7 @@ void LLDiskCache::createCache()
     {
         LLFile::mkdir(absl::StrCat(sCacheDir, gDirUtilp->getDirDelimiter(), prefixchar));
     }
+	prepopulateCacheWithStatic();
 }
 
 void LLDiskCache::purge()
@@ -116,30 +117,51 @@ void LLDiskCache::purge()
 
     LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL;
 
+    // Extra accounting to track the retention of static assets
+    int keep{0};
+    int del{0};
+    int skip{0};
     uintmax_t file_size_total = 0;
     for (const file_info_t& entry : file_info)
     {
         file_size_total += entry.second.first;
 
-        bool remove_file = false;
+        std::string action = "";
         if (file_size_total > mMaxSizeBytes)
         {
-            remove_file = true;
-            boost::system::error_code ec;
-            boost::filesystem::remove(entry.second.second, ec);
-            if (ec.failed())
+            action = "DELETE:";
+            auto uuid_as_string = LLUUID(gDirUtilp->getBaseFileName(entry.second.second.string(), true));
+            // LL_INFOS() << "checking UUID=" <<uuid_as_string<< LL_ENDL;
+            if (uuid_as_string.notNull() && mSkipList.find(uuid_as_string) != mSkipList.end())
             {
-                LL_WARNS() << "Failed to delete cached file " << entry.second.second << ": " << ec.message() << LL_ENDL;
-                continue;
+                // this is one of our protected items so no purging
+                action = "STATIC:";
+                skip++;
+                updateFileAccessTime(entry.second.second); // force these to the front of the list next time so that purge size works 
+            }
+            else 
+            {
+                del++;    // Extra accounting to track the retention of static assets
+
+                boost::filesystem::remove(entry.second.second, ec);
+                if (ec.failed())
+                {
+                    LL_WARNS() << "Failed to delete cache file " << entry.second.second << ": " << ec.message() << LL_ENDL;
+                }
             }
         }
+        else
+        {
+            keep++;
+            action = "  KEEP:";
+        }
 
         if (mEnableCacheDebugInfo)
         {
             // have to do this because of LL_INFO/LL_END weirdness
             std::ostringstream line;
 
-            line << (remove_file ? "DELETE:" : "  KEEP:" ) << "  ";
+            line << action << "  ";
             line << entry.first << "  ";
             line << entry.second.first << "  ";
             line << entry.second.second;
@@ -154,6 +176,7 @@ void LLDiskCache::purge()
         auto execute_time = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
         LL_INFOS() << "Total dir size after purge is " << dirFileSize(sCacheDir) << LL_ENDL;
         LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL;
+        LL_INFOS() << "Deleted: " << del << " Skipped: " << skip << " Kept: " << keep << LL_ENDL;
     }
 }
 
@@ -214,6 +237,48 @@ const std::string LLDiskCache::metaDataToFilepath(const LLUUID& id,
     return absl::StrCat(sCacheDir, dirdelim, std::string_view(&uuidstr[0], 1), dirdelim, uuidstr, sCacheFilenameExt);
 }
 
+void LLDiskCache::updateFileAccessTime(const boost::filesystem::path& file_path)
+{
+    /**
+     * Threshold in time_t units that is used to decide if the last access time
+     * time of the file is updated or not. Added as a precaution for the concern
+     * outlined in SL-14582  about frequent writes on older SSDs reducing their
+     * lifespan. I think this is the right place for the threshold value - rather
+     * than it being a pref - do comment on that Jira if you disagree...
+     *
+     * Let's start with 1 hour in time_t units and see how that unfolds
+     */
+    static const std::time_t time_threshold = 1 * 60 * 60;
+
+    // current time
+    const std::time_t cur_time = std::time(nullptr);
+
+    boost::system::error_code ec;
+
+    // file last write time
+    const std::time_t last_write_time = boost::filesystem::last_write_time(file_path, ec);
+    if (ec.failed())
+    {
+        LL_WARNS() << "Failed to read last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL;
+        return;
+    }
+
+    // delta between cur time and last time the file was written
+    const std::time_t delta_time = cur_time - last_write_time;
+
+    // we only write the new value if the time in time_threshold has elapsed
+    // before the last one
+    if (delta_time > time_threshold)
+    {
+        boost::filesystem::last_write_time(file_path, cur_time, ec);
+    }
+
+    if (ec.failed())
+    {
+        LL_WARNS() << "Failed to update last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL;
+    }
+}
+
 const std::string LLDiskCache::getCacheInfo()
 {
     uintmax_t cache_used_mb = dirFileSize(sCacheDir) / (1024U * 1024U);
@@ -223,6 +288,55 @@ const std::string LLDiskCache::getCacheInfo()
     return llformat("%juMB / %juMB (%.1f%% used)", cache_used_mb, max_in_mb, percent_used);
 }
 
+// Copy static items into cache and add to the skip list that prevents their purging
+// Note that there is no de-duplication nor other validation of the list.
+void LLDiskCache::prepopulateCacheWithStatic()
+{
+    LL_INFOS() << "Prepopulating cache with static assets" << LL_ENDL;
+
+    mSkipList.clear();
+
+    std::vector<std::string> from_folders;
+    from_folders.emplace_back(gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, "static_assets"));
+
+    for (const auto& from_folder : from_folders)
+    {
+        if (gDirUtilp->fileExists(from_folder))
+        {
+            auto assets_to_copy = gDirUtilp->getFilesInDir(from_folder);
+            for (auto from_asset_file : assets_to_copy)
+            {
+                from_asset_file = from_folder + gDirUtilp->getDirDelimiter() + from_asset_file;
+                // we store static assets as UUID.asset_type the asset_type is not used in the current simple cache format
+                auto uuid_as_string = LLUUID(gDirUtilp->getBaseFileName(from_asset_file, true));
+                if (uuid_as_string.notNull())
+                {
+                    auto to_asset_file = metaDataToFilepath(LLUUID(uuid_as_string), LLAssetType::AT_UNKNOWN);
+                    if (!gDirUtilp->fileExists(to_asset_file))
+                    {
+                        if (mEnableCacheDebugInfo)
+                        {
+                            LL_INFOS("LLDiskCache") << "Copying static asset " << from_asset_file << " to cache from " << from_folder << LL_ENDL;
+                        }
+                        if (!LLFile::copy(from_asset_file, to_asset_file))
+                        {
+                            LL_WARNS("LLDiskCache") << "Failed to copy " << from_asset_file << " to " << to_asset_file << LL_ENDL;
+                        }
+                    }
+                    if (mSkipList.find(uuid_as_string) == mSkipList.end())
+                    {
+                        if (mEnableCacheDebugInfo)
+                        {
+                            LL_INFOS("LLDiskCache") << "Adding " << uuid_as_string << " to skip list" << LL_ENDL;
+                        }
+                        mSkipList.insert(uuid_as_string);
+                    }
+                }
+            }
+        }
+    }
+}
+
 void LLDiskCache::clearCache()
 {
     /**

indra/llfilesystem/lldiskcache.h

@@ -64,6 +64,8 @@
 
 #include "llsingleton.h"
 
+#include "absl/container/flat_hash_set.h"
+
 class LLDiskCache final :
     public LLParamSingleton<LLDiskCache>
 {
@@ -107,12 +109,21 @@ class LLDiskCache final :
         static const std::string metaDataToFilepath(const LLUUID& id,
                                              LLAssetType::EType at);
 
+        /**
+         * Update the "last write time" of a file to "now". This must be called whenever a
+         * file in the cache is read (not written) so that the last time the file was
+         * accessed is up to date (This is used in the mechanism for purging the cache)
+         */
+        void updateFileAccessTime(const boost::filesystem::path& file_path);
         /**
          * Purge the oldest items in the cache so that the combined size of all files
          * is no bigger than mMaxSizeBytes.
          */
         void purge();
 
+        // copy from distribution into cache to replace static content
+        void prepopulateCacheWithStatic();
+
         /**
          * Clear the cache by removing all the files in the specified cache
          * directory individually. Only the files that contain a prefix defined
@@ -176,6 +187,8 @@ class LLDiskCache final :
          * various parts of the code
          */
         bool mEnableCacheDebugInfo;
+        
+        absl::flat_hash_set<LLUUID> mSkipList;
 };
 
 #endif // _LLDISKCACHE

indra/newview/app_settings/static_assets/02862ac0-a61d-94c2-5d11-5f5d319b1256.clothing

+LLWearable version 22
+RASL SOCKS BROWN
+
+	permissions 0
+	{
+		base_mask	7fffffff
+		owner_mask	7fffffff
+		group_mask	00000000
+		everyone_mask	00000000
+		next_owner_mask	00082000
+		creator_id	79908808-d272-49a8-864a-d133a10931e9
+		owner_id	79908808-d272-49a8-864a-d133a10931e9
+		last_owner_id	79908808-d272-49a8-864a-d133a10931e9
+		group_id	00000000-0000-0000-0000-000000000000
+	}
+	sale_info	0
+	{
+		sale_type	not
+		sale_price	10
+	}
+type 7
+parameters 4
+617 1
+818 1
+819 1
+820 1
+textures 1
+12 301a3097-2858-7614-829d-c5859741246f

indra/newview/app_settings/static_assets/02c4eae7-b6a3-b4b0-44b0-b7b29d7f913f.clothing

+LLWearable version 22
+RASL F SHOP SHOES FOOTSHAPER
+
+	permissions 0
+	{
+		base_mask	7fffffff
+		owner_mask	7fffffff
+		group_mask	00000000
+		everyone_mask	00000000
+		next_owner_mask	00082000
+		creator_id	df110c10-72e6-40d6-8916-14b4b0366b18
+		owner_id	df110c10-72e6-40d6-8916-14b4b0366b18
+		last_owner_id	df110c10-72e6-40d6-8916-14b4b0366b18
+		group_id	00000000-0000-0000-0000-000000000000
+	}
+	sale_info	0
+	{
+		sale_type	not
+		sale_price	10
+	}
+type 6
+parameters 10
+198 1
+503 .17
+508 -1
+513 0
+514 .37
+616 .1
+654 .08
+812 1
+813 1
+817 1
+textures 1
+7 3ab7e2fa-9572-ef36-1a30-d855dbea4f92

indra/newview/app_settings/static_assets/07d61cde-bc5b-f95f-f747-2e57eab4ef12.clothing

+LLWearable version 22
+RASL F LEARN LEGGINGS
+
+	permissions 0
+	{
+		base_mask	7fffffff
+		owner_mask	7fffffff
+		group_mask	00000000
+		everyone_mask	00000000
+		next_owner_mask	00082000
+		creator_id	79908808-d272-49a8-864a-d133a10931e9
+		owner_id	79908808-d272-49a8-864a-d133a10931e9
+		last_owner_id	79908808-d272-49a8-864a-d133a10931e9
+		group_id	00000000-0000-0000-0000-000000000000
+	}
+	sale_info	0
+	{
+		sale_type	not
+		sale_price	10
+	}
+type 11
+parameters 5
+619 1
+624 1
+824 1
+825 1
+826 1
+textures 1
+17 6f5d29ce-548f-fdd0-b8e1-1f90a123b76e

indra/newview/app_settings/static_assets/0b29c4fd-9c3c-3047-fcb4-addd3e42a289.clothing

+LLWearable version 22
+RASL M EXPLORE TANK UP (WHITE)
+
+	permissions 0
+	{
+		base_mask	7fffffff
+		owner_mask	7fffffff
+		group_mask	00000000
+		everyone_mask	00000000
+		next_owner_mask	00082000
+		creator_id	79908808-d272-49a8-864a-d133a10931e9
+		owner_id	79908808-d272-49a8-864a-d133a10931e9
+		last_owner_id	79908808-d272-49a8-864a-d133a10931e9
+		group_id	00000000-0000-0000-0000-000000000000
+	}
+	sale_info	0
+	{
+		sale_type	not
+		sale_price	10
+	}
+type 4
+parameters 10
+781 1
+800 1
+801 1
+802 1
+803 1
+804 1
+805 1
+828 0
+840 0
+868 0
+textures 1
+1 5ab7802c-a722-8cf0-66c6-6160efe4a130

indra/newview/app_settings/static_assets/0c5f60ae-1a54-1ee8-398d-7ed278552132.bodypart

+LLWearable version 22
+RASL M CREATE SHAPE (MIKO)
+
+	permissions 0
+	{
+		base_mask	7fffffff
+		owner_mask	7fffffff
+		group_mask	00000000
+		everyone_mask	00000000
+		next_owner_mask	00082000
+		creator_id	79908808-d272-49a8-864a-d133a10931e9
+		owner_id	79908808-d272-49a8-864a-d133a10931e9
+		last_owner_id	79908808-d272-49a8-864a-d133a10931e9
+		group_id	00000000-0000-0000-0000-000000000000
+	}
+	sale_info	0
+	{
+		sale_type	not
+		sale_price	10
+	}
+type 0
+parameters 82
+1 .21
+2 -.21
+4 .26
+5 -.07
+6 .13
+7 -.04
+8 -.2
+10 .25
+11 .1
+12 .07
+13 .03
+14 .67
+15 -.2
+17 .14
+18 .5
+19 -.4
+20 -.04
+21 .26
+22 .02
+23 -.06
+24 -.1
+25 .53
+27 .43
+33 1.14
+34 -.02
+35 0
+36 -.9
+37 -.62
+38 .02
+80 1
+105 .5
+155 -.24
+157 .02
+185 -.6
+193 .65
+196 -1.07
+505 .61
+506 0
+507 0
+515 -1
+517 -.13
+518 -.19
+629 .43
+637 .03
+646 -.06
+647 .03
+649 .5
+650 .65
+652 .45
+653 .05
+656 0
+659 .62
+662 .5
+663 0
+664 0
+665 .08
+675 -.14
+676 -.4
+678 .41
+682 .59
+683 -.12
+684 0
+685 .19
+690 .41
+692 .36
+693 .2
+753 .65
+756 -.62
+758 -.54
+759 .45
+760 .11
+764 .44
+765 .29
+769 .47
+773 .43
+795 .23
+796 .21
+799 .52
+841 0
+842 -.24
+879 .08
+880 -.9
+textures 0