From b08125a5874a89ce5210f8fb2c961ae17fb80fde Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Thu, 14 Jun 2012 16:31:48 -0400
Subject: [PATCH] LLMutex recursive lock, global & per-request tracing, simple
GET request, LLProxy support, HttpOptions starting to work, HTTP resource
waiting fixed. Non-LLThread-based threads need to do some registration or
LLMutex locks taken out in these threads will not work as expected (SH-3154).
We'll get a better solution later, this fixes some things for now. Tracing
of operations now supported. Global and per-request (via HttpOptions)
tracing levels of [0..3]. The 2 and 3 levels use libcurl's VERBOSE mode
combined with CURLOPT_DEBUGFUNCTION to stream high levels of detail into the
log. *Very* laggy but useful. Simple GET request supported (no Range:
header). Really just a degenrate case of a ranged get but supplied an API
anyway. Global option to use the LLProxy interface to setup CURL handles for
either socks5 or http proxy usage. This isn't really the most encapsulated
way to do this but a better solution will have to come later. The
wantHeaders and tracing options are now supported in HttpOptions giving
per-request controls. Big refactoring of the HTTP resource waiter in
lltexturefetch. What I was doing before wasn't correct. Instead, I'm
implementing the resource wait after the Semaphore model (though not using
system semaphores). So instead of having a sequence like: SEND_HTTP_REQ ->
WAIT_HTTP_RESOURCE -> SEND_HTTP_REQ, we now do WAIT_HTTP_RESOURCE ->
WAIT_HTTP_RESOURCE2 (actual wait) -> SEND_HTTP_REQ. Works well but the
prioritized filling of the corehttp library needs some performance work
later.
---
indra/llcommon/llthread.cpp | 7 +
indra/llcommon/llthread.h | 5 +
indra/llcorehttp/_httplibcurl.cpp | 24 +-
indra/llcorehttp/_httpoperation.cpp | 15 +-
indra/llcorehttp/_httpoperation.h | 4 +
indra/llcorehttp/_httpoprequest.cpp | 271 +++++++++++++++-----
indra/llcorehttp/_httpoprequest.h | 16 +-
indra/llcorehttp/_httpopsetget.cpp | 4 +-
indra/llcorehttp/_httppolicy.cpp | 21 +-
indra/llcorehttp/_httppolicy.h | 5 +-
indra/llcorehttp/_httppolicyglobal.cpp | 52 ++--
indra/llcorehttp/_httppolicyglobal.h | 6 +-
indra/llcorehttp/_httpservice.cpp | 18 +-
indra/llcorehttp/httpoptions.cpp | 18 +-
indra/llcorehttp/httpoptions.h | 14 +-
indra/llcorehttp/httprequest.cpp | 27 ++
indra/llcorehttp/httprequest.h | 75 +++++-
indra/llcorehttp/tests/test_httprequest.hpp | 228 +++++++++++++++-
indra/newview/llappviewer.cpp | 20 +-
indra/newview/lltexturefetch.cpp | 120 +++++----
indra/newview/lltexturefetch.h | 9 +
indra/newview/lltextureview.cpp | 5 +-
22 files changed, 803 insertions(+), 161 deletions(-)
diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp
index a6ad6b125c1..b27b64b26f4 100644
--- a/indra/llcommon/llthread.cpp
+++ b/indra/llcommon/llthread.cpp
@@ -71,6 +71,13 @@ LL_COMMON_API void assert_main_thread()
}
}
+void LLThread::registerThreadID()
+{
+#if !LL_DARWIN
+ sThreadID = ++sIDIter;
+#endif
+}
+
//
// Handed to the APR thread creation function
//
diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h
index b52e70ab2eb..54af41ec599 100644
--- a/indra/llcommon/llthread.h
+++ b/indra/llcommon/llthread.h
@@ -88,6 +88,11 @@ class LL_COMMON_API LLThread
U32 getID() const { return mID; }
+ // Called by threads *not* created via LLThread to register some
+ // internal state used by LLMutex. You must call this once early
+ // in the running thread to prevent collisions with the main thread.
+ static void registerThreadID();
+
private:
BOOL mPaused;
diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index e134a284017..a176dd5b2a2 100644
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -159,6 +159,17 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
op->mCurlActive = true;
+ if (op->mTracing > 0)
+ {
+ HttpPolicy & policy(mService->getPolicy());
+
+ LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle: "
+ << static_cast<HttpHandle>(op)
+ << ", Actives: " << mActiveOps.size()
+ << ", Readies: " << policy.getReadyCount(op->mReqPolicy)
+ << LL_ENDL;
+ }
+
// On success, make operation active
mActiveOps.insert(op);
}
@@ -190,10 +201,9 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
// Deactivate request
op->mCurlActive = false;
- // Set final status of request
+ // Set final status of request if it hasn't failed by other mechanisms yet
if (op->mStatus)
{
- // Only set if it hasn't failed by other mechanisms yet
op->mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, status);
}
if (op->mStatus)
@@ -209,6 +219,16 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
curl_easy_cleanup(handle);
op->mCurlHandle = NULL;
+ // Tracing
+ if (op->mTracing > 0)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: "
+ << static_cast<HttpHandle>(op)
+ << ", Status: " << op->mStatus.toHex()
+ << LL_ENDL;
+ }
+
+ // Dispatch to next stage
HttpPolicy & policy(mService->getPolicy());
bool still_active(policy.stageAfterCompletion(op));
diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp
index b5c58013d40..5a31bf90e74 100644
--- a/indra/llcorehttp/_httpoperation.cpp
+++ b/indra/llcorehttp/_httpoperation.cpp
@@ -34,6 +34,8 @@
#include "_httpreplyqueue.h"
#include "_httpservice.h"
+#include "lltimer.h"
+
namespace LLCore
{
@@ -49,8 +51,10 @@ HttpOperation::HttpOperation()
mReplyQueue(NULL),
mUserHandler(NULL),
mReqPolicy(HttpRequest::DEFAULT_POLICY_ID),
- mReqPriority(0U)
+ mReqPriority(0U),
+ mTracing(0)
{
+ mMetricCreated = totalTime();
}
@@ -113,7 +117,7 @@ void HttpOperation::stageFromActive(HttpService *)
llassert_always(false);
}
-
+
void HttpOperation::visitNotifier(HttpRequest *)
{
if (mUserHandler)
@@ -138,6 +142,13 @@ HttpStatus HttpOperation::cancel()
void HttpOperation::addAsReply()
{
+ if (mTracing > 0)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: "
+ << static_cast<HttpHandle>(this)
+ << LL_ENDL;
+ }
+
if (mReplyQueue)
{
addRef();
diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h
index c93aa2def92..de4939a0aca 100644
--- a/indra/llcorehttp/_httpoperation.h
+++ b/indra/llcorehttp/_httpoperation.h
@@ -110,6 +110,10 @@ class HttpOperation : public LLCoreInt::RefCounted
// Reply Data
HttpStatus mStatus;
+
+ // Tracing, debug and metrics
+ HttpTime mMetricCreated;
+ int mTracing;
}; // end class HttpOperation
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index e2550d057e0..f78971d8f27 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -63,6 +63,16 @@ int parse_content_range_header(char * buffer,
unsigned int * last,
unsigned int * length);
+
+// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and
+// escape and format it for a tracing line in logging. Absolutely
+// anything including NULs can be in the data. If @scrub is true,
+// non-printing or non-ascii characters are replaced with spaces
+// otherwise a %XX form of escaping is used.
+void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
+ std::string & safe_line);
+
+
#if defined(WIN32)
// Not available on windows where the legacy strtok interface
@@ -78,11 +88,6 @@ namespace LLCore
{
-// ==================================
-// HttpOpRequest
-// ==================================
-
-
HttpOpRequest::HttpOpRequest()
: HttpOperation(),
mProcFlags(0U),
@@ -237,6 +242,19 @@ HttpStatus HttpOpRequest::cancel()
}
+HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id,
+ HttpRequest::priority_t priority,
+ const std::string & url,
+ HttpOptions * options,
+ HttpHeaders * headers)
+{
+ setupCommon(policy_id, priority, url, NULL, options, headers);
+ mReqMethod = HOR_GET;
+
+ return HttpStatus();
+}
+
+
HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
HttpRequest::priority_t priority,
const std::string & url,
@@ -245,30 +263,16 @@ HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
HttpOptions * options,
HttpHeaders * headers)
{
- HttpStatus status;
-
- mProcFlags = 0;
- mReqPolicy = policy_id;
- mReqPriority = priority;
+ setupCommon(policy_id, priority, url, NULL, options, headers);
mReqMethod = HOR_GET;
- mReqURL = url;
mReqOffset = offset;
mReqLength = len;
if (offset || len)
{
mProcFlags |= PF_SCAN_RANGE_HEADER;
}
- if (headers && ! mReqHeaders)
- {
- headers->addRef();
- mReqHeaders = headers;
- }
- if (options && ! mReqOptions)
- {
- mReqOptions = new HttpOptions(*options);
- }
- return status;
+ return HttpStatus();
}
@@ -279,29 +283,10 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id,
HttpOptions * options,
HttpHeaders * headers)
{
- HttpStatus status;
-
- mProcFlags = 0;
- mReqPolicy = policy_id;
- mReqPriority = priority;
+ setupCommon(policy_id, priority, url, body, options, headers);
mReqMethod = HOR_POST;
- mReqURL = url;
- if (body)
- {
- body->addRef();
- mReqBody = body;
- }
- if (headers && ! mReqHeaders)
- {
- headers->addRef();
- mReqHeaders = headers;
- }
- if (options && ! mReqOptions)
- {
- mReqOptions = new HttpOptions(*options);
- }
- return status;
+ return HttpStatus();
}
@@ -312,12 +297,23 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
HttpOptions * options,
HttpHeaders * headers)
{
- HttpStatus status;
+ setupCommon(policy_id, priority, url, body, options, headers);
+ mReqMethod = HOR_PUT;
+
+ return HttpStatus();
+}
+
+void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
+ HttpRequest::priority_t priority,
+ const std::string & url,
+ BufferArray * body,
+ HttpOptions * options,
+ HttpHeaders * headers)
+{
mProcFlags = 0;
mReqPolicy = policy_id;
mReqPriority = priority;
- mReqMethod = HOR_PUT;
mReqURL = url;
if (body)
{
@@ -331,10 +327,14 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
}
if (options && ! mReqOptions)
{
- mReqOptions = new HttpOptions(*options);
+ options->addRef();
+ mReqOptions = options;
+ if (options->getWantHeaders())
+ {
+ mProcFlags |= PF_SAVE_HEADERS;
+ }
+ mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3));
}
-
- return status;
}
@@ -394,30 +394,29 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
const std::string * opt_value(NULL);
- if (policy.get(HttpRequest::GP_CA_PATH, opt_value))
+ long opt_long(0L);
+ policy.get(HttpRequest::GP_LLPROXY, &opt_long);
+ if (opt_long)
{
- curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+ // Use the viewer-based thread-safe API which has a
+ // fast/safe check for proxy enable. Would like to
+ // encapsulate this someway...
+ LLProxy::getInstance()->applyProxySettings(mCurlHandle);
}
- if (policy.get(HttpRequest::GP_CA_FILE, opt_value))
+ else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value))
{
- curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
+ // *TODO: This is fine for now but get fuller socks/
+ // authentication thing going later....
+ curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
+ curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
}
- if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value))
+ if (policy.get(HttpRequest::GP_CA_PATH, &opt_value))
{
- if (*opt_value == "LLProxy")
- {
- // Use the viewer-based thread-safe API which has a
- // fast/safe check for proxy enable. Would like to
- // encapsulate this someway...
- LLProxy::getInstance()->applyProxySettings(mCurlHandle);
- }
- else
- {
- // *TODO: This is fine for now but get fuller socks/
- // authentication thing going later....
- curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
- curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
- }
+ curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+ }
+ if (policy.get(HttpRequest::GP_CA_FILE, &opt_value))
+ {
+ curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
}
switch (mReqMethod)
@@ -463,6 +462,14 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
// *FIXME: fail out here
break;
}
+
+ // Tracing
+ if (mTracing > 1)
+ {
+ curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
+ curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle);
+ curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
+ }
// There's a CURLOPT for this now...
if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
@@ -621,6 +628,101 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
return hdr_size;
}
+
+int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata)
+{
+ HttpOpRequest * op(NULL);
+ curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
+ // *FIXME: check the pointer
+
+ std::string safe_line;
+ std::string tag;
+ bool logit(false);
+ len = (std::min)(len, size_t(256)); // Keep things reasonable in all cases
+
+ switch (info)
+ {
+ case CURLINFO_TEXT:
+ if (op->mTracing > 1)
+ {
+ tag = "TEXT";
+ escape_libcurl_debug_data(buffer, len, true, safe_line);
+ logit = true;
+ }
+ break;
+
+ case CURLINFO_HEADER_IN:
+ if (op->mTracing > 1)
+ {
+ tag = "HEADERIN";
+ escape_libcurl_debug_data(buffer, len, true, safe_line);
+ logit = true;
+ }
+ break;
+
+ case CURLINFO_HEADER_OUT:
+ if (op->mTracing > 1)
+ {
+ tag = "HEADEROUT";
+ escape_libcurl_debug_data(buffer, len, true, safe_line);
+ logit = true;
+ }
+ break;
+
+ case CURLINFO_DATA_IN:
+ if (op->mTracing > 1)
+ {
+ tag = "DATAIN";
+ logit = true;
+ if (op->mTracing > 2)
+ {
+ escape_libcurl_debug_data(buffer, len, false, safe_line);
+ }
+ else
+ {
+ std::ostringstream out;
+ out << len << " Bytes";
+ safe_line = out.str();
+ }
+ }
+ break;
+
+ case CURLINFO_DATA_OUT:
+ if (op->mTracing > 1)
+ {
+ tag = "DATAOUT";
+ logit = true;
+ if (op->mTracing > 2)
+ {
+ escape_libcurl_debug_data(buffer, len, false, safe_line);
+ }
+ else
+ {
+ std::ostringstream out;
+ out << len << " Bytes";
+ safe_line = out.str();
+ }
+ }
+ break;
+
+ default:
+ logit = false;
+ break;
+ }
+
+ if (logit)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle: "
+ << static_cast<HttpHandle>(op)
+ << ", Type: " << tag
+ << ", Data: " << safe_line
+ << LL_ENDL;
+ }
+
+ return 0;
+}
+
+
} // end namespace LLCore
@@ -694,6 +796,43 @@ char *strtok_r(char *str, const char *delim, char ** savestate)
#endif
+
+void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
+{
+ std::string out;
+ len = (std::min)(len, size_t(200));
+ out.reserve(3 * len);
+ for (int i(0); i < len; ++i)
+ {
+ unsigned char uc(static_cast<unsigned char>(buffer[i]));
+
+ if (uc < 32 || uc > 126)
+ {
+ if (scrub)
+ {
+ out.append(1, ' ');
+ }
+ else
+ {
+ static const char hex[] = "0123456789ABCDEF";
+ char convert[4];
+
+ convert[0] = '%';
+ convert[1] = hex[(uc >> 4) % 16];
+ convert[2] = hex[uc % 16];
+ convert[3] = '\0';
+ out.append(convert);
+ }
+ }
+ else
+ {
+ out.append(1, buffer[i]);
+ }
+ }
+ safe_line.swap(out);
+}
+
+
} // end anonymous namespace
diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h
index 80893beb40f..fc2301057c9 100644
--- a/indra/llcorehttp/_httpoprequest.h
+++ b/indra/llcorehttp/_httpoprequest.h
@@ -76,6 +76,12 @@ class HttpOpRequest : public HttpOperation
public:
// Setup Methods
+ HttpStatus setupGet(HttpRequest::policy_t policy_id,
+ HttpRequest::priority_t priority,
+ const std::string & url,
+ HttpOptions * options,
+ HttpHeaders * headers);
+
HttpStatus setupGetByteRange(HttpRequest::policy_t policy_id,
HttpRequest::priority_t priority,
const std::string & url,
@@ -103,15 +109,23 @@ class HttpOpRequest : public HttpOperation
virtual HttpStatus cancel();
protected:
+ void setupCommon(HttpRequest::policy_t policy_id,
+ HttpRequest::priority_t priority,
+ const std::string & url,
+ BufferArray * body,
+ HttpOptions * options,
+ HttpHeaders * headers);
+
static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata);
static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata);
static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata);
+ static int debugCallback(CURL *, curl_infotype info, char * buffer, size_t len, void * userdata);
protected:
unsigned int mProcFlags;
static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U;
static const unsigned int PF_SAVE_HEADERS = 0x00000002U;
-
+
public:
// Request data
EMethod mReqMethod;
diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp
index 21e058b2be5..c1357f9ae51 100644
--- a/indra/llcorehttp/_httpopsetget.cpp
+++ b/indra/llcorehttp/_httpopsetget.cpp
@@ -89,8 +89,8 @@ void HttpOpSetGet::stageFromRequest(HttpService * service)
}
if (mStatus)
{
- const std::string * value;
- if ((mStatus = pol_opt.get(setting, value)))
+ const std::string * value(NULL);
+ if ((mStatus = pol_opt.get(setting, &value)))
{
mStrValue = *value;
}
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 8ee3f886583..0e08d882760 100644
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -107,6 +107,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries
<< " being scheduled for " << delta << " uSecs from now."
<< LL_ENDL;
+ if (op->mTracing > 0)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: "
+ << static_cast<HttpHandle>(op)
+ << LL_ENDL;
+ }
mState[policy_class].mRetryQueue.push(op);
}
@@ -224,8 +230,8 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
}
else if (op->mPolicyRetries)
{
- LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries."
- << LL_ENDL;
+ LL_DEBUGS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries."
+ << LL_ENDL;
}
op->stageFromActive(mService);
@@ -234,4 +240,15 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
}
+int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class)
+{
+ if (policy_class < HttpRequest::POLICY_CLASS_LIMIT)
+ {
+ return (mState[policy_class].mReadyQueue.size()
+ + mState[policy_class].mRetryQueue.size());
+ }
+ return 0;
+}
+
+
} // end namespace LLCore
diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h
index 73c22bab789..4114f648488 100644
--- a/indra/llcorehttp/_httppolicy.h
+++ b/indra/llcorehttp/_httppolicy.h
@@ -97,7 +97,10 @@ class HttpPolicy
}
void setPolicies(const HttpPolicyGlobal & global);
-
+
+ // Get ready counts for a particular class
+ int getReadyCount(HttpRequest::policy_t policy_class);
+
protected:
struct State
{
diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp
index d95d73cfba0..6b1de38fd64 100644
--- a/indra/llcorehttp/_httppolicyglobal.cpp
+++ b/indra/llcorehttp/_httppolicyglobal.cpp
@@ -33,7 +33,9 @@ namespace LLCore
HttpPolicyGlobal::HttpPolicyGlobal()
: mSetMask(0UL),
- mConnectionLimit(32L)
+ mConnectionLimit(32L),
+ mTrace(0),
+ mUseLLProxy(0)
{}
@@ -50,6 +52,8 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other)
mCAPath = other.mCAPath;
mCAFile = other.mCAFile;
mHttpProxy = other.mHttpProxy;
+ mTrace = other.mTrace;
+ mUseLLProxy = other.mUseLLProxy;
}
return *this;
}
@@ -63,6 +67,14 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value)
mConnectionLimit = value;
break;
+ case HttpRequest::GP_TRACE:
+ mTrace = llclamp(value, 0L, 3L);
+ break;
+
+ case HttpRequest::GP_LLPROXY:
+ mUseLLProxy = llclamp(value, 0L, 1L);
+ break;
+
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
@@ -97,54 +109,64 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri
}
-HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long & value)
+HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value)
{
static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
+ long * src(NULL);
switch (opt)
{
case HttpRequest::GP_CONNECTION_LIMIT:
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
- value = mConnectionLimit;
+ src = &mConnectionLimit;
+ break;
+
+ case HttpRequest::GP_TRACE:
+ src = &mTrace;
+ break;
+
+ case HttpRequest::GP_LLPROXY:
+ src = &mUseLLProxy;
break;
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
+ if (! (mSetMask & (1UL << int(opt))))
+ return not_set;
+
+ *value = *src;
return HttpStatus();
}
-HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string *& value)
+HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value)
{
static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
+ const std::string * src(NULL);
switch (opt)
{
case HttpRequest::GP_CA_PATH:
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
- value = &mCAPath;
+ src = &mCAPath;
break;
case HttpRequest::GP_CA_FILE:
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
- value = &mCAFile;
+ src = &mCAFile;
break;
case HttpRequest::GP_HTTP_PROXY:
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
- value = &mHttpProxy;
+ src = &mHttpProxy;
break;
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
+ if (! (mSetMask & (1UL << int(opt))))
+ return not_set;
+
+ *value = src;
return HttpStatus();
}
diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h
index f4bb4d4b258..a50d0e41887 100644
--- a/indra/llcorehttp/_httppolicyglobal.h
+++ b/indra/llcorehttp/_httppolicyglobal.h
@@ -48,8 +48,8 @@ class HttpPolicyGlobal
public:
HttpStatus set(HttpRequest::EGlobalPolicy opt, long value);
HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value);
- HttpStatus get(HttpRequest::EGlobalPolicy opt, long & value);
- HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string *& value);
+ HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value);
+ HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value);
public:
unsigned long mSetMask;
@@ -57,6 +57,8 @@ class HttpPolicyGlobal
std::string mCAPath;
std::string mCAFile;
std::string mHttpProxy;
+ long mTrace;
+ long mUseLLProxy;
}; // end class HttpPolicyGlobal
} // end namespace LLCore
diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp
index 920a3f3b6d0..beba8f08f4f 100644
--- a/indra/llcorehttp/_httpservice.cpp
+++ b/indra/llcorehttp/_httpservice.cpp
@@ -36,6 +36,7 @@
#include "_thread.h"
#include "lltimer.h"
+#include "llthread.h"
// Tuning parameters
@@ -186,8 +187,10 @@ void HttpService::shutdown()
void HttpService::threadRun(LLCoreInt::HttpThread * thread)
{
boost::this_thread::disable_interruption di;
- ELoopSpeed loop(REQUEST_SLEEP);
+
+ LLThread::registerThreadID();
+ ELoopSpeed loop(REQUEST_SLEEP);
while (! mExitRequested)
{
loop = processRequestQueue(loop);
@@ -226,6 +229,19 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop)
// Process operation
if (! mExitRequested)
{
+ // Setup for subsequent tracing
+ long tracing(0);
+ mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing);
+ op->mTracing = (std::max)(op->mTracing, int(tracing));
+
+ if (op->mTracing > 0)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: "
+ << static_cast<HttpHandle>(op)
+ << LL_ENDL;
+ }
+
+ // Stage
op->stageFromRequest(this);
}
diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp
index 15b505f5bfd..155fbda7f1d 100644
--- a/indra/llcorehttp/httpoptions.cpp
+++ b/indra/llcorehttp/httpoptions.cpp
@@ -33,13 +33,15 @@ namespace LLCore
HttpOptions::HttpOptions()
: RefCounted(true),
- mWantHeaders(false)
+ mWantHeaders(false),
+ mTracing(0)
{}
HttpOptions::HttpOptions(const HttpOptions & rhs)
: RefCounted(true),
- mWantHeaders(rhs.mWantHeaders)
+ mWantHeaders(rhs.mWantHeaders),
+ mTracing(rhs.mTracing)
{}
@@ -47,4 +49,16 @@ HttpOptions::~HttpOptions()
{}
+void HttpOptions::setWantHeaders()
+{
+ mWantHeaders = true;
+}
+
+
+void HttpOptions::setTrace(long level)
+{
+ mTracing = int(level);
+}
+
+
} // end namespace LLCore
diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h
index 267a982dd5e..0b9dfdc1de2 100644
--- a/indra/llcorehttp/httpoptions.h
+++ b/indra/llcorehttp/httpoptions.h
@@ -67,11 +67,21 @@ class HttpOptions : public LLCoreInt::RefCounted
void operator=(const HttpOptions &); // Not defined
public:
+ void setWantHeaders();
+ bool getWantHeaders() const
+ {
+ return mWantHeaders;
+ }
+
+ void setTrace(long level);
+ int getTrace() const
+ {
+ return mTracing;
+ }
protected:
- // *TODO: add some options
bool mWantHeaders;
-
+ long int mTracing;
}; // end class HttpOptions
diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp
index 089eee76f33..2036ecfd1c4 100644
--- a/indra/llcorehttp/httprequest.cpp
+++ b/indra/llcorehttp/httprequest.cpp
@@ -135,6 +135,33 @@ HttpStatus HttpRequest::getStatus() const
}
+HttpHandle HttpRequest::requestGet(policy_t policy_id,
+ priority_t priority,
+ const std::string & url,
+ HttpOptions * options,
+ HttpHeaders * headers,
+ HttpHandler * user_handler)
+{
+ HttpStatus status;
+ HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID);
+
+ HttpOpRequest * op = new HttpOpRequest();
+ if (! (status = op->setupGet(policy_id, priority, url, options, headers)))
+ {
+ op->release();
+ mLastReqStatus = status;
+ return handle;
+ }
+ op->setReplyPath(mReplyQueue, user_handler);
+ mRequestQueue->addOp(op); // transfers refcount
+
+ mLastReqStatus = status;
+ handle = static_cast<HttpHandle>(op);
+
+ return handle;
+}
+
+
HttpHandle HttpRequest::requestGetByteRange(policy_t policy_id,
priority_t priority,
const std::string & url,
diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h
index a953aa28d0a..4e78ed37190 100644
--- a/indra/llcorehttp/httprequest.h
+++ b/indra/llcorehttp/httprequest.h
@@ -111,10 +111,43 @@ class HttpRequest
/// Maximum number of connections the library will use to
/// perform operations. This is somewhat soft as the underlying
/// transport will cache some connections (up to 5).
- GP_CONNECTION_LIMIT, ///< Takes long giving number of connections
- GP_CA_PATH, ///< System path/directory where SSL certs are stored.
- GP_CA_FILE, ///< System path/file containing certs.
- GP_HTTP_PROXY ///< String giving host/port to use for HTTP proxy
+
+ /// A long value setting the maximum number of connections
+ /// allowed over all policy classes. Note that this will be
+ /// a somewhat soft value. There may be an additional five
+ /// connections per policy class depending upon runtime
+ /// behavior.
+ GP_CONNECTION_LIMIT,
+
+ /// String containing a system-appropriate directory name
+ /// where SSL certs are stored.
+ GP_CA_PATH,
+
+ /// String giving a full path to a file containing SSL certs.
+ GP_CA_FILE,
+
+ /// String of host/port to use as simple HTTP proxy. This is
+ /// going to change in the future into something more elaborate
+ /// that may support richer schemes.
+ GP_HTTP_PROXY,
+
+ /// Long value that if non-zero enables the use of the
+ /// traditional LLProxy code for http/socks5 support. If
+ /// enabled, has priority over GP_HTTP_PROXY.
+ GP_LLPROXY,
+
+ /// Long value setting the logging trace level for the
+ /// library. Possible values are:
+ /// 0 - No tracing (default)
+ /// 1 - Basic tracing of request start, stop and major events.
+ /// 2 - Connection, header and payload size information from
+ /// HTTP transactions.
+ /// 3 - Partial logging of payload itself.
+ ///
+ /// These values are also used in the trace modes for
+ /// individual requests in HttpOptions. Also be aware that
+ /// tracing tends to impact performance of the viewer.
+ GP_TRACE
};
/// Set a parameter on a global policy option. Calls
@@ -133,7 +166,7 @@ class HttpRequest
/// the class in other methods. If -1, an error
/// occurred and @see getStatus() may provide more
/// detail on the reason.
- policy_t createPolicyClass();
+ static policy_t createPolicyClass();
enum EClassPolicy
{
@@ -157,7 +190,7 @@ class HttpRequest
/// @param opt Enum of option to be set.
/// @param value Desired value of option.
/// @return Standard status code.
- HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
+ static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
/// @}
@@ -176,6 +209,36 @@ class HttpRequest
///
HttpStatus getStatus() const;
+ /// Queue a full HTTP GET request to be issued for entire entity.
+ /// The request is queued and serviced by the working thread and
+ /// notification of completion delivered to the optional HttpHandler
+ /// argument during @see update() calls.
+ ///
+ /// With a valid handle returned, it can be used to reference the
+ /// request in other requests (like cancellation) and will be an
+ /// argument when any HttpHandler object is invoked.
+ ///
+ /// @param policy_id Default or user-defined policy class under
+ /// which this request is to be serviced.
+ /// @param priority Standard priority scheme inherited from
+ /// Indra code base (U32-type scheme).
+ /// @param url
+ /// @param options (optional)
+ /// @param headers (optional)
+ /// @param handler (optional)
+ /// @return The handle of the request if successfully
+ /// queued or LLCORE_HTTP_HANDLE_INVALID if the
+ /// request could not be queued. In the latter
+ /// case, @see getStatus() will return more info.
+ ///
+ HttpHandle requestGet(policy_t policy_id,
+ priority_t priority,
+ const std::string & url,
+ HttpOptions * options,
+ HttpHeaders * headers,
+ HttpHandler * handler);
+
+
/// Queue a full HTTP GET request to be issued with a 'Range' header.
/// The request is queued and serviced by the working thread and
/// notification of completion delivered to the optional HttpHandler
diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp
index 61698f34d84..5b04796c8ad 100644
--- a/indra/llcorehttp/tests/test_httprequest.hpp
+++ b/indra/llcorehttp/tests/test_httprequest.hpp
@@ -395,7 +395,7 @@ void HttpRequestTestObjectType::test<5>()
{
ScopedCurlInit ready;
- set_test_name("HttpRequest GET + Stop execution");
+ set_test_name("HttpRequest GET to dead port + Stop execution");
// Handler can be stack-allocated *if* there are no dangling
// references to it after completion of this method.
@@ -496,6 +496,7 @@ void HttpRequestTestObjectType::test<5>()
}
}
+
template <> template <>
void HttpRequestTestObjectType::test<6>()
{
@@ -517,6 +518,114 @@ void HttpRequestTestObjectType::test<6>()
HttpRequest * req = NULL;
+ try
+ {
+ // Get singletons created
+ HttpRequest::createService();
+
+ // Start threading early so that thread memory is invariant
+ // over the test.
+ HttpRequest::startThread();
+
+ // create a new ref counted object with an implicit reference
+ req = new HttpRequest();
+ ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
+
+ // Issue a GET that *can* connect
+ mStatus = HttpStatus(200);
+ HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID,
+ 0U,
+ url_base,
+ NULL,
+ NULL,
+ &handler);
+ ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump.
+ int count(0);
+ int limit(10);
+ while (count++ < limit && mHandlerCalls < 1)
+ {
+ req->update(1000);
+ usleep(100000);
+ }
+ ensure("Request executed in reasonable time", count < limit);
+ ensure("One handler invocation for request", mHandlerCalls == 1);
+
+ // Okay, request a shutdown of the servicing thread
+ mStatus = HttpStatus();
+ handle = req->requestStopThread(&handler);
+ ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump again
+ count = 0;
+ limit = 10;
+ while (count++ < limit && mHandlerCalls < 2)
+ {
+ req->update(1000);
+ usleep(100000);
+ }
+ ensure("Second request executed in reasonable time", count < limit);
+ ensure("Second handler invocation", mHandlerCalls == 2);
+
+ // See that we actually shutdown the thread
+ count = 0;
+ limit = 10;
+ while (count++ < limit && ! HttpService::isStopped())
+ {
+ usleep(100000);
+ }
+ ensure("Thread actually stopped running", HttpService::isStopped());
+
+ // release the request object
+ delete req;
+ req = NULL;
+
+ // Shut down service
+ HttpRequest::destroyService();
+
+ ensure("Two handler calls on the way out", 2 == mHandlerCalls);
+
+#if defined(WIN32)
+ // Can only do this memory test on Windows. On other platforms,
+ // the LL logging system holds on to memory and produces what looks
+ // like memory leaks...
+
+ // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal());
+ ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal());
+#endif
+ }
+ catch (...)
+ {
+ stop_thread(req);
+ delete req;
+ HttpRequest::destroyService();
+ throw;
+ }
+}
+
+
+template <> template <>
+void HttpRequestTestObjectType::test<7>()
+{
+ ScopedCurlInit ready;
+
+ std::string url_base(get_base_url());
+ std::cerr << "Base: " << url_base << std::endl;
+
+ set_test_name("HttpRequest GET with Range: header to real service");
+
+ // Handler can be stack-allocated *if* there are no dangling
+ // references to it after completion of this method.
+ // Create before memory record as the string copy will bump numbers.
+ TestHandler2 handler(this, "handler");
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+ mHandlerCalls = 0;
+
+ HttpRequest * req = NULL;
+
try
{
// Get singletons created
@@ -605,8 +714,9 @@ void HttpRequestTestObjectType::test<6>()
}
}
+
template <> template <>
-void HttpRequestTestObjectType::test<7>()
+void HttpRequestTestObjectType::test<8>()
{
ScopedCurlInit ready;
@@ -725,7 +835,7 @@ void HttpRequestTestObjectType::test<7>()
}
template <> template <>
-void HttpRequestTestObjectType::test<8>()
+void HttpRequestTestObjectType::test<9>()
{
ScopedCurlInit ready;
@@ -843,6 +953,118 @@ void HttpRequestTestObjectType::test<8>()
}
}
+template <> template <>
+void HttpRequestTestObjectType::test<10>()
+{
+ ScopedCurlInit ready;
+
+ std::string url_base(get_base_url());
+ std::cerr << "Base: " << url_base << std::endl;
+
+ set_test_name("HttpRequest GET with some tracing");
+
+ // Handler can be stack-allocated *if* there are no dangling
+ // references to it after completion of this method.
+ // Create before memory record as the string copy will bump numbers.
+ TestHandler2 handler(this, "handler");
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+ mHandlerCalls = 0;
+
+ HttpRequest * req = NULL;
+
+ try
+ {
+ // Get singletons created
+ HttpRequest::createService();
+
+ // Enable tracing
+ HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2);
+
+ // Start threading early so that thread memory is invariant
+ // over the test.
+ HttpRequest::startThread();
+
+ // create a new ref counted object with an implicit reference
+ req = new HttpRequest();
+ ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
+
+ // Issue a GET that *can* connect
+ mStatus = HttpStatus(200);
+ HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
+ 0U,
+ url_base,
+ 0,
+ 0,
+ NULL,
+ NULL,
+ &handler);
+ ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump.
+ int count(0);
+ int limit(10);
+ while (count++ < limit && mHandlerCalls < 1)
+ {
+ req->update(1000);
+ usleep(100000);
+ }
+ ensure("Request executed in reasonable time", count < limit);
+ ensure("One handler invocation for request", mHandlerCalls == 1);
+
+ // Okay, request a shutdown of the servicing thread
+ mStatus = HttpStatus();
+ handle = req->requestStopThread(&handler);
+ ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump again
+ count = 0;
+ limit = 10;
+ while (count++ < limit && mHandlerCalls < 2)
+ {
+ req->update(1000);
+ usleep(100000);
+ }
+ ensure("Second request executed in reasonable time", count < limit);
+ ensure("Second handler invocation", mHandlerCalls == 2);
+
+ // See that we actually shutdown the thread
+ count = 0;
+ limit = 10;
+ while (count++ < limit && ! HttpService::isStopped())
+ {
+ usleep(100000);
+ }
+ ensure("Thread actually stopped running", HttpService::isStopped());
+
+ // release the request object
+ delete req;
+ req = NULL;
+
+ // Shut down service
+ HttpRequest::destroyService();
+
+ ensure("Two handler calls on the way out", 2 == mHandlerCalls);
+
+#if defined(WIN32)
+ // Can only do this memory test on Windows. On other platforms,
+ // the LL logging system holds on to memory and produces what looks
+ // like memory leaks...
+
+ // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal());
+ ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal());
+#endif
+ }
+ catch (...)
+ {
+ stop_thread(req);
+ delete req;
+ HttpRequest::destroyService();
+ throw;
+ }
+}
+
} // end namespace tut
namespace
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index e2c13e77e3a..430dd89c3e4 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -5356,11 +5356,9 @@ void CoreHttp::init()
<< LL_ENDL;
}
- mRequest = new LLCore::HttpRequest;
-
// Point to our certs or SSH/https: will fail on connect
- status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE,
- gDirUtilp->getCAFile());
+ status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE,
+ gDirUtilp->getCAFile());
if (! status)
{
LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: "
@@ -5371,15 +5369,22 @@ void CoreHttp::init()
// Establish HTTP Proxy. "LLProxy" is a special string which directs
// the code to use LLProxy::applyProxySettings() to establish any
// HTTP or SOCKS proxy for http operations.
- status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_HTTP_PROXY,
- std::string("LLProxy"));
+ status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1);
if (! status)
{
LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: "
<< status.toString()
<< LL_ENDL;
}
-
+
+ // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy):
+ // 0 - None
+ // 1 - Basic start, stop simple transitions
+ // 2 - libcurl CURLOPT_VERBOSE mode with brief lines
+ // 3 - with partial data content
+ status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0);
+
+ // Kick the thread
status = LLCore::HttpRequest::startThread();
if (! status)
{
@@ -5388,6 +5393,7 @@ void CoreHttp::init()
<< LL_ENDL;
}
+ mRequest = new LLCore::HttpRequest;
}
diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp
index f5e7540e85e..664af02f78c 100644
--- a/indra/newview/lltexturefetch.cpp
+++ b/indra/newview/lltexturefetch.cpp
@@ -431,6 +431,22 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler
void lockWorkMutex() { mWorkMutex.lock(); }
void unlockWorkMutex() { mWorkMutex.unlock(); }
+ // Locks: Mw
+ void acquireHttpSemaphore()
+ {
+ llassert(! mHttpHasResource);
+ mHttpHasResource = true;
+ --mFetcher->mHttpSemaphore;
+ }
+
+ // Locks: Mw
+ void releaseHttpSemaphore()
+ {
+ llassert(mHttpHasResource);
+ mHttpHasResource = false;
+ ++mFetcher->mHttpSemaphore;
+ }
+
private:
enum e_state // mState
{
@@ -444,8 +460,9 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler
CACHE_POST,
LOAD_FROM_NETWORK,
LOAD_FROM_SIMULATOR,
- SEND_HTTP_REQ, // Commit to sending as HTTP
WAIT_HTTP_RESOURCE, // Waiting for HTTP resources
+ WAIT_HTTP_RESOURCE2, // Waiting for HTTP resources
+ SEND_HTTP_REQ, // Commit to sending as HTTP
WAIT_HTTP_REQ, // Request sent, wait for completion
DECODE_IMAGE,
DECODE_IMAGE_UPDATE,
@@ -532,7 +549,7 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler
bool mHttpActive; // Active request to http library
unsigned int mHttpReplySize;
unsigned int mHttpReplyOffset;
- bool mHttpReleased; // Has been released from resource wait once
+ bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore
};
//////////////////////////////////////////////////////////////////////////////
@@ -768,8 +785,9 @@ const char* LLTextureFetchWorker::sStateDescs[] = {
"CACHE_POST",
"LOAD_FROM_NETWORK",
"LOAD_FROM_SIMULATOR",
- "SEND_HTTP_REQ",
"WAIT_HTTP_RESOURCE",
+ "WAIT_HTTP_RESOURCE2",
+ "SEND_HTTP_REQ",
"WAIT_HTTP_REQ",
"DECODE_IMAGE",
"DECODE_IMAGE_UPDATE",
@@ -836,7 +854,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher,
mHttpActive(false),
mHttpReplySize(0U),
mHttpReplyOffset(0U),
- mHttpReleased(true)
+ mHttpHasResource(false)
{
mCanUseNET = mUrl.empty() ;
@@ -860,6 +878,10 @@ LLTextureFetchWorker::~LLTextureFetchWorker()
llassert_always(!haveWork());
lockWorkMutex(); // +Mw (should be useless)
+ if (mHttpHasResource)
+ {
+ releaseHttpSemaphore();
+ }
if (mHttpActive)
{
// Issue a cancel on a live request...
@@ -1126,7 +1148,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
llwarns << "Unknown URL Type: " << mUrl << llendl;
}
setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
- mState = SEND_HTTP_REQ;
+ mState = WAIT_HTTP_RESOURCE;
}
else
{
@@ -1223,7 +1245,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
}
if (mCanUseHTTP && !mUrl.empty())
{
- mState = SEND_HTTP_REQ;
+ mState = WAIT_HTTP_RESOURCE;
setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
if(mWriteToCacheState != NOT_WRITE)
{
@@ -1287,31 +1309,38 @@ bool LLTextureFetchWorker::doWork(S32 param)
return false;
}
- if (mState == SEND_HTTP_REQ)
+ if (mState == WAIT_HTTP_RESOURCE)
{
- if (! mCanUseHTTP)
- {
- return true; // abort
- }
-
// NOTE:
// control the number of the http requests issued for:
// 1, not openning too many file descriptors at the same time;
// 2, control the traffic of http so udp gets bandwidth.
//
- if (! mHttpReleased)
+ // If it looks like we're busy, keep this request here.
+ // Otherwise, advance into the HTTP states.
+ if (mFetcher->mHttpSemaphore <= 0 || mFetcher->getHttpWaitersCount())
{
- // If this request hasn't been released before and it looks like
- // we're busy, put this request into resource wait and allow something
- // else to come to the front.
- if (mFetcher->getNumHTTPRequests() >= HTTP_REQUESTS_IN_QUEUE_HIGH_WATER ||
- mFetcher->getHttpWaitersCount())
- {
- mState = WAIT_HTTP_RESOURCE;
- setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
- mFetcher->addHttpWaiter(this->mID);
- return false;
- }
+ mState = WAIT_HTTP_RESOURCE2;
+ setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
+ mFetcher->addHttpWaiter(this->mID);
+ return false;
+ }
+ mState = SEND_HTTP_REQ;
+ acquireHttpSemaphore();
+ }
+
+ if (mState == WAIT_HTTP_RESOURCE2)
+ {
+ // Just idle it if we make it to the head...
+ return false;
+ }
+
+ if (mState == SEND_HTTP_REQ)
+ {
+ if (! mCanUseHTTP)
+ {
+ releaseHttpSemaphore();
+ return true; // abort
}
mFetcher->removeFromNetworkQueue(this, false);
@@ -1327,10 +1356,12 @@ bool LLTextureFetchWorker::doWork(S32 param)
// We already have all the data, just decode it
mLoadedDiscard = mFormattedImage->getDiscardLevel();
mState = DECODE_IMAGE;
+ releaseHttpSemaphore();
return false;
}
else
{
+ releaseHttpSemaphore();
return true; // abort.
}
}
@@ -1365,6 +1396,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
{
llwarns << "HTTP GET request failed for " << mID << llendl;
resetFormattedData();
+ releaseHttpSemaphore();
return true; // failed
}
@@ -1377,13 +1409,6 @@ bool LLTextureFetchWorker::doWork(S32 param)
// fall through
}
- if (mState == WAIT_HTTP_RESOURCE)
- {
- // Nothing to do until releaseHttpWaiters() puts us back
- // into the flow...
- return false;
- }
-
if (mState == WAIT_HTTP_REQ)
{
if (mLoaded)
@@ -1401,6 +1426,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
mState = INIT;
mCanUseHTTP = false;
setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
+ releaseHttpSemaphore();
return false;
}
}
@@ -1423,12 +1449,14 @@ bool LLTextureFetchWorker::doWork(S32 param)
// Use available data
mLoadedDiscard = mFormattedImage->getDiscardLevel();
mState = DECODE_IMAGE;
+ releaseHttpSemaphore();
return false;
}
// Fail harder
resetFormattedData();
mState = DONE;
+ releaseHttpSemaphore();
return true; // failed
}
@@ -1443,6 +1471,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
// abort.
mState = DONE;
+ releaseHttpSemaphore();
return true;
}
@@ -1491,6 +1520,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
mWriteToCacheState = SHOULD_WRITE ;
}
setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
+ releaseHttpSemaphore();
return false;
}
else
@@ -2137,7 +2167,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image
mQAMode(qa_mode),
mHttpRequest(NULL),
mHttpOptions(NULL),
- mHttpHeaders(NULL)
+ mHttpHeaders(NULL),
+ mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER)
{
mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS");
mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold"));
@@ -3155,12 +3186,11 @@ void LLTextureFetch::removeHttpWaiter(const LLUUID & tid)
// Locks: -Mw (must not hold any worker when called)
void LLTextureFetch::releaseHttpWaiters()
{
- if (HTTP_REQUESTS_IN_QUEUE_LOW_WATER < getNumHTTPRequests())
+ if (mHttpSemaphore < HTTP_REQUESTS_IN_QUEUE_LOW_WATER)
return;
// Quickly make a copy of all the LLUIDs. Get off the
// mutex as early as possible.
-
typedef std::vector<LLUUID> uuid_vec_t;
uuid_vec_t tids;
@@ -3171,13 +3201,12 @@ void LLTextureFetch::releaseHttpWaiters()
return;
const size_t limit(mHttpWaitResource.size());
- tids.resize(limit);
- wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin());
- for (int i(0);
- i < limit && mHttpWaitResource.end() != iter;
- ++i, ++iter)
+ tids.reserve(limit);
+ for (wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin());
+ mHttpWaitResource.end() != iter;
+ ++iter)
{
- tids[i] = *iter;
+ tids.push_back(*iter);
}
} // -Mfnq
@@ -3196,28 +3225,29 @@ void LLTextureFetch::releaseHttpWaiters()
tids2.insert(worker);
}
}
+ tids.clear();
// Release workers up to the high water mark. Since we aren't
// holding any locks at this point, we can be in competition
// with other callers. Do defensive things like getting
// refreshed counts of requests and checking if someone else
// has moved any worker state around....
- tids.clear();
for (worker_set_t::iterator iter2(tids2.begin());
- tids2.end() != iter2 && 0 < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - getNumHTTPRequests());
+ tids2.end() != iter2 && mHttpSemaphore > 0;
++iter2)
{
LLTextureFetchWorker * worker(* iter2);
worker->lockWorkMutex(); // +Mw
- if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE != worker->mState)
+ if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE2 != worker->mState)
{
worker->unlockWorkMutex(); // -Mw
continue;
}
- worker->mHttpReleased = true;
+
worker->mState = LLTextureFetchWorker::SEND_HTTP_REQ;
worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority);
+ worker->acquireHttpSemaphore();
worker->unlockWorkMutex(); // -Mw
removeHttpWaiter(worker->mID);
@@ -3456,7 +3486,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher)
}
// In QA mode, Metrics submode, log the result for ease of testing
- if (fetcher->isQAMode() || true)
+ if (fetcher->isQAMode())
{
LL_INFOS("Textures") << merged_llsd << LL_ENDL;
}
diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h
index 4ee13d171e8..50e31816236 100644
--- a/indra/newview/lltexturefetch.h
+++ b/indra/newview/lltexturefetch.h
@@ -312,6 +312,15 @@ class LLTextureFetch : public LLWorkerThread
LLCore::HttpOptions * mHttpOptions; // Ttf
LLCore::HttpHeaders * mHttpHeaders; // Ttf
+ // We use a resource semaphore to keep HTTP requests in
+ // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the
+ // transport. This keeps them near where they can be cheaply
+ // reprioritized rather than dumping them all across a thread
+ // where it's more expensive to get at them. Requests in either
+ // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore
+ // and tracking state transitions is critical to liveness.
+ int mHttpSemaphore; // Ttf
+
typedef std::set<LLUUID> wait_http_res_queue_t;
wait_http_res_queue_t mHttpWaitResource; // Mfnq
diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp
index 5f1d7829ed1..bb1535d23db 100644
--- a/indra/newview/lltextureview.cpp
+++ b/indra/newview/lltextureview.cpp
@@ -234,15 +234,16 @@ void LLTextureBar::draw()
{ "DSK", LLColor4::blue }, // CACHE_POST
{ "NET", LLColor4::green }, // LOAD_FROM_NETWORK
{ "SIM", LLColor4::green }, // LOAD_FROM_SIMULATOR
+ { "HTW", LLColor4::green }, // WAIT_HTTP_RESOURCE
+ { "HTW", LLColor4::green }, // WAIT_HTTP_RESOURCE2
{ "REQ", LLColor4::yellow },// SEND_HTTP_REQ
- { "HTW", LLColor4::green }, // WAIT_HTTP_RES
{ "HTP", LLColor4::green }, // WAIT_HTTP_REQ
{ "DEC", LLColor4::yellow },// DECODE_IMAGE
{ "DEC", LLColor4::green }, // DECODE_IMAGE_UPDATE
{ "WRT", LLColor4::purple },// WRITE_TO_CACHE
{ "WRT", LLColor4::orange },// WAIT_ON_WRITE
{ "END", LLColor4::red }, // DONE
-#define LAST_STATE 13
+#define LAST_STATE 14
{ "CRE", LLColor4::magenta }, // LAST_STATE+1
{ "FUL", LLColor4::green }, // LAST_STATE+2
{ "BAD", LLColor4::red }, // LAST_STATE+3
--
GitLab