ESC-109 Write single-thread asset stats collector for wearable.

Code-complete with unit tests and foundation for other collectors.
Interestingly, sim and viewer have two different ideas about
asset type enumeration (compatible, one's just longer).  Both
are missing mesh though that's to be expected.

indra/newview/CMakeLists.txt

@@ -476,6 +476,7 @@ set(viewer_SOURCE_FILES
     llvectorperfoptions.cpp
     llversioninfo.cpp
     llviewchildren.cpp
+    llviewerassetstats.cpp
     llviewerassetstorage.cpp
     llviewerassettype.cpp
     llviewerattachmenu.cpp
@@ -1004,6 +1005,7 @@ set(viewer_HEADER_FILES
     llvectorperfoptions.h
     llversioninfo.h
     llviewchildren.h
+    llviewerassetstats.h
     llviewerassetstorage.h
     llviewerassettype.h
     llviewerattachmenu.h
@@ -1930,6 +1932,11 @@ if (LL_TESTS)
     "${test_libs}"
     )
 
+  LL_ADD_INTEGRATION_TEST(llviewerassetstats
+	llviewerassetstats.cpp
+    "${test_libs}"
+    )
+
   #ADD_VIEWER_BUILD_TEST(llmemoryview viewer)
   #ADD_VIEWER_BUILD_TEST(llagentaccess viewer)
   #ADD_VIEWER_BUILD_TEST(llworldmap viewer)

indra/newview/llviewerassetstats.cpp

+/** 
+ * @file llviewerassetstats.cpp
+ * @brief 
+ *
+ * $LicenseInfo:firstyear=2010&license=viewergpl$
+ * 
+ * Copyright (c) 2010, Linden Research, Inc.
+ * 
+ * Second Life Viewer Source Code
+ * The source code in this file ("Source Code") is provided by Linden Lab
+ * to you under the terms of the GNU General Public License, version 2.0
+ * ("GPL"), unless you have obtained a separate licensing agreement
+ * ("Other License"), formally executed by you and Linden Lab.  Terms of
+ * the GPL can be found in doc/GPL-license.txt in this distribution, or
+ * online at http://secondlifegrid.net/programs/open_source/licensing/gplv2
+ * 
+ * There are special exceptions to the terms and conditions of the GPL as
+ * it is applied to this Source Code. View the full text of the exception
+ * in the file doc/FLOSS-exception.txt in this software distribution, or
+ * online at
+ * http://secondlifegrid.net/programs/open_source/licensing/flossexception
+ * 
+ * By copying, modifying or distributing this software, you acknowledge
+ * that you have read and understood your obligations described above,
+ * and agree to abide by those obligations.
+ * 
+ * ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO
+ * WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY,
+ * COMPLETENESS OR PERFORMANCE.
+ * $/LicenseInfo$
+ */
+
+#include "llviewerassetstats.h"
+
+#include "stdtypes.h"
+
+/*
+ * References
+ *
+ * Project:
+ *   <TBD>
+ *
+ * Test Plan:
+ *   <TBD>
+ *
+ * Jiras:
+ *   <TBD>
+ *
+ * Unit Tests:
+ *   <TBD>
+ *
+ */
+
+
+// ------------------------------------------------------
+// Global data definitions
+// ------------------------------------------------------
+LLViewerAssetStats * gViewerAssetStats = NULL;
+
+
+// ------------------------------------------------------
+// Local declarations
+// ------------------------------------------------------
+namespace
+{
+
+static LLViewerAssetStats::EViewerAssetCategories
+asset_type_to_category(const LLViewerAssetType::EType at);
+
+}
+
+// ------------------------------------------------------
+// LLViewerAssetStats class definition
+// ------------------------------------------------------
+LLViewerAssetStats::LLViewerAssetStats()
+{
+	reset();
+}
+
+
+void
+LLViewerAssetStats::reset()
+{
+	for (int i = 0; i < LL_ARRAY_SIZE(mRequests); ++i)
+	{
+		mRequests[i].mEnqueued.reset();
+		mRequests[i].mDequeued.reset();
+		mRequests[i].mResponse.reset();
+	}
+}
+
+void
+LLViewerAssetStats::recordGetEnqueued(LLViewerAssetType::EType at)
+{
+	const EViewerAssetCategories eac(asset_type_to_category(at));
+	
+	++mRequests[int(eac)].mEnqueued;
+}
+	
+void
+LLViewerAssetStats::recordGetDequeued(LLViewerAssetType::EType at)
+{
+	const EViewerAssetCategories eac(asset_type_to_category(at));
+
+	++mRequests[int(eac)].mDequeued;
+}
+
+void
+LLViewerAssetStats::recordGetServiced(LLViewerAssetType::EType at, F64 duration)
+{
+	const EViewerAssetCategories eac(asset_type_to_category(at));
+
+	mRequests[int(eac)].mResponse.record(duration);
+}
+
+const LLSD
+LLViewerAssetStats::asLLSD() const
+{
+	// Top-level tags
+	static const LLSD::String tags[EVACCount] = 
+		{
+			LLSD::String("get_texture"),
+			LLSD::String("get_wearable"),
+			LLSD::String("get_sound"),
+			LLSD::String("get_gesture"),
+			LLSD::String("get_other")
+		};
+
+	// Sub-tags
+	static const LLSD::String enq_tag("enqueued");
+	static const LLSD::String deq_tag("dequeued");
+	static const LLSD::String rcnt_tag("resp_count");
+	static const LLSD::String rmin_tag("resp_min");
+	static const LLSD::String rmax_tag("resp_max");
+	static const LLSD::String rmean_tag("resp_mean");
+	
+	LLSD ret = LLSD::emptyMap();
+
+	for (int i = 0; i < EVACCount; ++i)
+	{
+		LLSD & slot = ret[tags[i]];
+		slot = LLSD::emptyMap();
+		slot[enq_tag] = LLSD(S32(mRequests[i].mEnqueued.getCount()));
+		slot[deq_tag] = LLSD(S32(mRequests[i].mDequeued.getCount()));
+		slot[rcnt_tag] = LLSD(S32(mRequests[i].mResponse.getCount()));
+		slot[rmin_tag] = LLSD(mRequests[i].mResponse.getMin());
+		slot[rmax_tag] = LLSD(mRequests[i].mResponse.getMax());
+		slot[rmean_tag] = LLSD(mRequests[i].mResponse.getMean());
+	}
+
+	return ret;
+}
+
+// ------------------------------------------------------
+// Global free-function definitions (LLViewerAssetStatsFF namespace)
+// ------------------------------------------------------
+
+namespace LLViewerAssetStatsFF
+{
+
+void
+record_enqueue(LLViewerAssetType::EType at)
+{
+	if (! gViewerAssetStats)
+		return;
+
+	gViewerAssetStats->recordGetEnqueued(at);
+}
+
+void
+record_dequeue(LLViewerAssetType::EType at)
+{
+	if (! gViewerAssetStats)
+		return;
+
+	gViewerAssetStats->recordGetDequeued(at);
+}
+
+void
+record_response(LLViewerAssetType::EType at, F64 duration)
+{
+	if (! gViewerAssetStats)
+		return;
+
+	gViewerAssetStats->recordGetServiced(at, duration);
+}
+
+} // namespace LLViewerAssetStatsFF
+
+
+// ------------------------------------------------------
+// Local function definitions
+// ------------------------------------------------------
+
+namespace
+{
+
+LLViewerAssetStats::EViewerAssetCategories
+asset_type_to_category(const LLViewerAssetType::EType at)
+{
+	// For statistical purposes, we divide GETs into several
+	// populations of asset fetches:
+	//  - textures which are de-prioritized in the asset system
+	//  - wearables (clothing, bodyparts) which directly affect
+	//    user experiences when they log in
+	//  - sounds
+	//  - gestures
+	//  - everything else.
+	//
+	llassert_always(26 == LLViewerAssetType::AT_COUNT);
+
+	// Multiple asset definitions are floating around so this requires some
+	// maintenance and attention.
+	static const LLViewerAssetStats::EViewerAssetCategories asset_to_bin_map[LLViewerAssetType::AT_COUNT] =
+		{
+			LLViewerAssetStats::EVACTextureGet,				// (0) AT_TEXTURE
+			LLViewerAssetStats::EVACSoundGet,				// AT_SOUND
+			LLViewerAssetStats::EVACOtherGet,				// AT_CALLINGCARD
+			LLViewerAssetStats::EVACOtherGet,				// AT_LANDMARK
+			LLViewerAssetStats::EVACOtherGet,				// AT_SCRIPT
+			LLViewerAssetStats::EVACWearableGet,			// AT_CLOTHING
+			LLViewerAssetStats::EVACOtherGet,				// AT_OBJECT
+			LLViewerAssetStats::EVACOtherGet,				// AT_NOTECARD
+			LLViewerAssetStats::EVACOtherGet,				// AT_CATEGORY
+			LLViewerAssetStats::EVACOtherGet,				// AT_ROOT_CATEGORY
+			LLViewerAssetStats::EVACOtherGet,				// (10) AT_LSL_TEXT
+			LLViewerAssetStats::EVACOtherGet,				// AT_LSL_BYTECODE
+			LLViewerAssetStats::EVACOtherGet,				// AT_TEXTURE_TGA
+			LLViewerAssetStats::EVACWearableGet,			// AT_BODYPART
+			LLViewerAssetStats::EVACOtherGet,				// AT_TRASH
+			LLViewerAssetStats::EVACOtherGet,				// AT_SNAPSHOT_CATEGORY
+			LLViewerAssetStats::EVACOtherGet,				// AT_LOST_AND_FOUND
+			LLViewerAssetStats::EVACSoundGet,				// AT_SOUND_WAV
+			LLViewerAssetStats::EVACOtherGet,				// AT_IMAGE_TGA
+			LLViewerAssetStats::EVACOtherGet,				// AT_IMAGE_JPEG
+			LLViewerAssetStats::EVACGestureGet,				// (20) AT_ANIMATION
+			LLViewerAssetStats::EVACGestureGet,				// AT_GESTURE
+			LLViewerAssetStats::EVACOtherGet,				// AT_SIMSTATE
+			LLViewerAssetStats::EVACOtherGet,				// AT_FAVORITE
+			LLViewerAssetStats::EVACOtherGet,				// AT_LINK
+			LLViewerAssetStats::EVACOtherGet,				// AT_LINK_FOLDER
+#if 0
+			// When LLViewerAssetType::AT_COUNT == 49
+			LLViewerAssetStats::EVACOtherGet,				// AT_FOLDER_ENSEMBLE_START
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// (30)
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// (40)
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// 
+			LLViewerAssetStats::EVACOtherGet,				// AT_FOLDER_ENSEMBLE_END
+			LLViewerAssetStats::EVACOtherGet,				// AT_CURRENT_OUTFIT
+			LLViewerAssetStats::EVACOtherGet,				// AT_OUTFIT
+			LLViewerAssetStats::EVACOtherGet				// AT_MY_OUTFITS
+#endif
+		};
+	
+	if (at < 0 || at >= LLViewerAssetType::AT_COUNT)
+	{
+		return LLViewerAssetStats::EVACOtherGet;
+	}
+	return asset_to_bin_map[at];
+}
+
+} // anonymous namespace

indra/newview/llviewerassetstats.h

+/** 
+ * @file llviewerassetstats.h
+ * @brief Client-side collection of asset request statistics
+ *
+ * $LicenseInfo:firstyear=2010&license=viewergpl$
+ * 
+ * Copyright (c) 2010, Linden Research, Inc.
+ * 
+ * Second Life Viewer Source Code
+ * The source code in this file ("Source Code") is provided by Linden Lab
+ * to you under the terms of the GNU General Public License, version 2.0
+ * ("GPL"), unless you have obtained a separate licensing agreement
+ * ("Other License"), formally executed by you and Linden Lab.  Terms of
+ * the GPL can be found in doc/GPL-license.txt in this distribution, or
+ * online at http://secondlifegrid.net/programs/open_source/licensing/gplv2
+ * 
+ * There are special exceptions to the terms and conditions of the GPL as
+ * it is applied to this Source Code. View the full text of the exception
+ * in the file doc/FLOSS-exception.txt in this software distribution, or
+ * online at
+ * http://secondlifegrid.net/programs/open_source/licensing/flossexception
+ * 
+ * By copying, modifying or distributing this software, you acknowledge
+ * that you have read and understood your obligations described above,
+ * and agree to abide by those obligations.
+ * 
+ * ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO
+ * WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY,
+ * COMPLETENESS OR PERFORMANCE.
+ * $/LicenseInfo$
+ */
+
+#ifndef LL_LLVIEWERASSETSTATUS_H
+#define	LL_LLVIEWERASSETSTATUS_H
+
+
+#include "linden_common.h"
+
+#include "llviewerassettype.h"
+#include "llviewerassetstorage.h"
+#include "llsimplestat.h"
+#include "llsd.h"
+
+/**
+ * @class LLViewerAssetStats
+ * @brief Records events and performance of asset put/get operations.
+ *
+ * The asset system is a combination of common code and server-
+ * and viewer-overridden derivations.  The common code is presented
+ * in here as the 'front-end' and deriviations (really the server)
+ * are presented as 'back-end'.  The distinction isn't perfect as
+ * there are legacy asset transfer systems which mostly appear
+ * as front-end stats.
+ *
+ * Statistics collected are fairly basic:
+ *  - Counts of enqueue and dequeue operations
+ *  - Counts of duplicated request fetches
+ *  - Min/Max/Mean of asset transfer operations
+ *
+ * While the stats collection interfaces appear to be fairly
+ * orthogonal across methods (GET, PUT) and asset types (texture,
+ * bodypart, etc.), the actual internal collection granularity
+ * varies greatly.  GET's operations found in the cache are
+ * treated as a single group as are duplicate requests.  Non-
+ * cached items are broken down into three groups:  textures,
+ * wearables (bodyparts, clothing) and the rest.  PUT operations
+ * are broken down into two categories:  temporary assets and
+ * non-temp.  Back-end operations do not distinguish asset types,
+ * only GET, PUT (temp) and PUT (non-temp).
+ * 
+ * No coverage for Estate Assets or Inventory Item Assets which use
+ * some different interface conventions.  It could be expanded to cover
+ * them.
+ *
+ * Access to results is by conversion to an LLSD with some standardized
+ * key names.  The intent of this structure is to be emitted as
+ * standard syslog-based metrics formatting where it can be picked
+ * up by interested parties.
+ *
+ * For convenience, a set of free functions in namespace LLAssetStatsFF
+ * are provided which operate on various counters in a way that
+ * is highly-compatible with the simulator code.
+ */
+class LLViewerAssetStats
+{
+public:
+	LLViewerAssetStats();
+	// Default destructor and assignment operator are correct.
+	
+	enum EViewerAssetCategories
+	{
+		EVACTextureGet,			//< Texture GETs
+		EVACWearableGet,		//< Wearable GETs
+		EVACSoundGet,			//< Sound GETs
+		EVACGestureGet,			//< Gesture GETs
+		EVACOtherGet,			//< Other GETs
+		
+		EVACCount				// Must be last
+	};
+	
+	void reset();
+
+	// Non-Cached GET Requests
+	void recordGetEnqueued(LLViewerAssetType::EType at);
+	void recordGetDequeued(LLViewerAssetType::EType at);
+	void recordGetServiced(LLViewerAssetType::EType at, F64 duration);
+
+	// Report Generation
+	const LLSD asLLSD() const;
+	
+protected:
+
+	struct 
+	{
+		LLSimpleStatCounter		mEnqueued;
+		LLSimpleStatCounter		mDequeued;
+		LLSimpleStatMMM<>		mResponse;
+	} mRequests [EVACCount];
+};
+
+
+/**
+ * Expectation is that the simulator and other asset-handling
+ * code will create a single instance of the stats class and
+ * make it available here.  The free functions examine this
+ * for non-zero and perform their functions conditionally.  The
+ * instance methods themselves make no assumption about this.
+ */
+extern LLViewerAssetStats * gViewerAssetStats;
+
+namespace LLViewerAssetStatsFF
+{
+
+void record_enqueue(LLViewerAssetType::EType at);
+
+void record_dequeue(LLViewerAssetType::EType at);
+
+void record_response(LLViewerAssetType::EType at, F64 duration);
+
+} // namespace LLViewerAssetStatsFF
+
+
+#endif	// LL_LLVIEWERASSETSTATUS_H

indra/newview/tests/llviewerassetstats_test.cpp

+/** 
+ * @file llviewerassetstats_tut.cpp
+ * @date 2010-10-28
+ * @brief Test cases for some of newview/llviewerassetstats.cpp
+ *
+ * $LicenseInfo:firstyear=2010&license=viewergpl$
+ * 
+ * Copyright (c) 2010, Linden Research, Inc.
+ * 
+ * Second Life Viewer Source Code
+ * The source code in this file ("Source Code") is provided by Linden Lab
+ * to you under the terms of the GNU General Public License, version 2.0
+ * ("GPL"), unless you have obtained a separate licensing agreement
+ * ("Other License"), formally executed by you and Linden Lab.  Terms of
+ * the GPL can be found in doc/GPL-license.txt in this distribution, or
+ * online at http://secondlifegrid.net/programs/open_source/licensing/gplv2
+ * 
+ * There are special exceptions to the terms and conditions of the GPL as
+ * it is applied to this Source Code. View the full text of the exception
+ * in the file doc/FLOSS-exception.txt in this software distribution, or
+ * online at
+ * http://secondlifegrid.net/programs/open_source/licensing/flossexception
+ * 
+ * By copying, modifying or distributing this software, you acknowledge
+ * that you have read and understood your obligations described above,
+ * and agree to abide by those obligations.
+ * 
+ * ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO
+ * WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY,
+ * COMPLETENESS OR PERFORMANCE.
+ * $/LicenseInfo$
+ */
+
+#include "linden_common.h"
+
+#include <tut/tut.hpp>
+#include <iostream>
+
+#include "lltut.h"
+#include "../llviewerassetstats.h"
+
+static const char * all_keys[] = 
+{
+	"get_other",
+	"get_texture",
+	"get_wearable",
+	"get_sound",
+	"get_gesture"
+};
+
+static const char * resp_keys[] = 
+{
+	"get_other",
+	"get_texture",
+	"get_wearable",
+	"get_sound",
+	"get_gesture"
+};
+
+static const char * sub_keys[] =
+{
+	"dequeued",
+	"enqueued",
+	"resp_count",
+	"resp_max",
+	"resp_min",
+	"resp_mean"
+};
+
+namespace tut
+{
+	struct tst_viewerassetstats_index
+	{};
+	typedef test_group<tst_viewerassetstats_index> tst_viewerassetstats_index_t;
+	typedef tst_viewerassetstats_index_t::object tst_viewerassetstats_index_object_t;
+	tut::tst_viewerassetstats_index_t tut_tst_viewerassetstats_index("tst_viewerassetstats_test");
+
+	// Testing free functions without global stats allocated
+	template<> template<>
+	void tst_viewerassetstats_index_object_t::test<1>()
+	{
+		// Check that helpers aren't bothered by missing global stats
+		ensure("Global gViewerAssetStats should be NULL", (NULL == gViewerAssetStats));
+
+		LLViewerAssetStatsFF::record_enqueue(LLViewerAssetType::AT_TEXTURE);
+
+		LLViewerAssetStatsFF::record_dequeue(LLViewerAssetType::AT_TEXTURE);
+
+		LLViewerAssetStatsFF::record_response(LLViewerAssetType::AT_GESTURE, 12.3);
+	}
+
+	// Create a non-global instance and check the structure
+	template<> template<>
+	void tst_viewerassetstats_index_object_t::test<2>()
+	{
+		ensure("Global gViewerAssetStats should be NULL", (NULL == gViewerAssetStats));
+
+		LLViewerAssetStats * it = new LLViewerAssetStats();
+
+		ensure("Global gViewerAssetStats should still be NULL", (NULL == gViewerAssetStats));
+		
+		LLSD sd = it->asLLSD();
+		
+		delete it;
+
+		// Check the structure of the LLSD
+		for (int i = 0; i < LL_ARRAY_SIZE(all_keys); ++i)
+		{
+			std::string line = llformat("Has '%s' key", all_keys[i]);
+			ensure(line, sd.has(all_keys[i]));
+		}
+
+		for (int i = 0; i < LL_ARRAY_SIZE(resp_keys); ++i)
+		{
+			for (int j = 0; j < LL_ARRAY_SIZE(sub_keys); ++j)
+			{
+				std::string line = llformat("Key '%s' has '%s' key", resp_keys[i], sub_keys[j]);
+				ensure(line, sd[resp_keys[i]].has(sub_keys[j]));
+			}
+		}
+	}
+
+	// Create a non-global instance and check some content
+	template<> template<>
+	void tst_viewerassetstats_index_object_t::test<3>()
+	{
+		LLViewerAssetStats * it = new LLViewerAssetStats();
+		
+		LLSD sd = it->asLLSD();
+		
+		delete it;
+
+		// Check a few points on the tree for content
+		ensure("sd[get_texture][dequeued] is 0", (0 == sd["get_texture"]["dequeued"].asInteger()));
+		ensure("sd[get_sound][resp_min] is 0", (0.0 == sd["get_sound"]["resp_min"].asReal()));
+	}
+
+	// Create a global instance and verify free functions do something useful
+	template<> template<>
+	void tst_viewerassetstats_index_object_t::test<4>()
+	{
+		gViewerAssetStats = new LLViewerAssetStats();
+
+		LLViewerAssetStatsFF::record_enqueue(LLViewerAssetType::AT_TEXTURE);
+		LLViewerAssetStatsFF::record_dequeue(LLViewerAssetType::AT_TEXTURE);
+
+		LLViewerAssetStatsFF::record_enqueue(LLViewerAssetType::AT_BODYPART);
+		LLViewerAssetStatsFF::record_dequeue(LLViewerAssetType::AT_BODYPART);
+
+		LLSD sd = gViewerAssetStats->asLLSD();
+		
+		// Check a few points on the tree for content
+		ensure("sd[get_texture][enqueued] is 1", (1 == sd["get_texture"]["enqueued"].asInteger()));
+		ensure("sd[get_gesture][dequeued] is 0", (0 == sd["get_gesture"]["dequeued"].asInteger()));
+
+		// Reset and check zeros...
+		gViewerAssetStats->reset();
+		sd = gViewerAssetStats->asLLSD();
+		
+		delete gViewerAssetStats;
+		gViewerAssetStats = NULL;
+
+		ensure("sd[get_texture][enqueued] is reset", (0 == sd["get_texture"]["enqueued"].asInteger()));
+		ensure("sd[get_gesture][dequeued] is reset", (0 == sd["get_gesture"]["dequeued"].asInteger()));
+	}
+
+}