diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp
index a6ad6b125c1ad14cd1a9370f0713ed2159be655c..b27b64b26f46564c5364d2586030923b2eef208b 100644
--- a/indra/llcommon/llthread.cpp
+++ b/indra/llcommon/llthread.cpp
@@ -71,6 +71,13 @@ LL_COMMON_API void assert_main_thread()
}
}
+void LLThread::registerThreadID()
+{
+#if !LL_DARWIN
+ sThreadID = ++sIDIter;
+#endif
+}
+
//
// Handed to the APR thread creation function
//
diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h
index b52e70ab2ebcb637ac5fed388ba3b9ef285abb0e..54af41ec5994dc45f9fdee6f595786dc7a205bd0 100644
--- a/indra/llcommon/llthread.h
+++ b/indra/llcommon/llthread.h
@@ -88,6 +88,11 @@ class LL_COMMON_API LLThread
U32 getID() const { return mID; }
+ // Called by threads *not* created via LLThread to register some
+ // internal state used by LLMutex. You must call this once early
+ // in the running thread to prevent collisions with the main thread.
+ static void registerThreadID();
+
private:
BOOL mPaused;
diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index e134a28401750dc965fadf07d08e53d658a96087..a176dd5b2a2c5e2458dd7283f7a0c6d5c1a38407 100644
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -159,6 +159,17 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
op->mCurlActive = true;
+ if (op->mTracing > 0)
+ {
+ HttpPolicy & policy(mService->getPolicy());
+
+ LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle: "
+ << static_cast<HttpHandle>(op)
+ << ", Actives: " << mActiveOps.size()
+ << ", Readies: " << policy.getReadyCount(op->mReqPolicy)
+ << LL_ENDL;
+ }
+
// On success, make operation active
mActiveOps.insert(op);
}
@@ -190,10 +201,9 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
// Deactivate request
op->mCurlActive = false;
- // Set final status of request
+ // Set final status of request if it hasn't failed by other mechanisms yet
if (op->mStatus)
{
- // Only set if it hasn't failed by other mechanisms yet
op->mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, status);
}
if (op->mStatus)
@@ -209,6 +219,16 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
curl_easy_cleanup(handle);
op->mCurlHandle = NULL;
+ // Tracing
+ if (op->mTracing > 0)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: "
+ << static_cast<HttpHandle>(op)
+ << ", Status: " << op->mStatus.toHex()
+ << LL_ENDL;
+ }
+
+ // Dispatch to next stage
HttpPolicy & policy(mService->getPolicy());
bool still_active(policy.stageAfterCompletion(op));
diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp
index b5c58013d4066114a04c863fcbe49789960e79ad..5a31bf90e741d1141be309b5ce16e9f4b4051f29 100644
--- a/indra/llcorehttp/_httpoperation.cpp
+++ b/indra/llcorehttp/_httpoperation.cpp
@@ -34,6 +34,8 @@
#include "_httpreplyqueue.h"
#include "_httpservice.h"
+#include "lltimer.h"
+
namespace LLCore
{
@@ -49,8 +51,10 @@ HttpOperation::HttpOperation()
mReplyQueue(NULL),
mUserHandler(NULL),
mReqPolicy(HttpRequest::DEFAULT_POLICY_ID),
- mReqPriority(0U)
+ mReqPriority(0U),
+ mTracing(0)
{
+ mMetricCreated = totalTime();
}
@@ -113,7 +117,7 @@ void HttpOperation::stageFromActive(HttpService *)
llassert_always(false);
}
-
+
void HttpOperation::visitNotifier(HttpRequest *)
{
if (mUserHandler)
@@ -138,6 +142,13 @@ HttpStatus HttpOperation::cancel()
void HttpOperation::addAsReply()
{
+ if (mTracing > 0)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: "
+ << static_cast<HttpHandle>(this)
+ << LL_ENDL;
+ }
+
if (mReplyQueue)
{
addRef();
diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h
index c93aa2def923c5acfc71528425a3842a4f6f41f1..de4939a0aca9743dc0c775b07d745d24fb51ea8f 100644
--- a/indra/llcorehttp/_httpoperation.h
+++ b/indra/llcorehttp/_httpoperation.h
@@ -110,6 +110,10 @@ class HttpOperation : public LLCoreInt::RefCounted
// Reply Data
HttpStatus mStatus;
+
+ // Tracing, debug and metrics
+ HttpTime mMetricCreated;
+ int mTracing;
}; // end class HttpOperation
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index e2550d057e0282c285221b126ba24f9074b50fe0..f78971d8f275c175922a19f68e12172a87305ca2 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -63,6 +63,16 @@ int parse_content_range_header(char * buffer,
unsigned int * last,
unsigned int * length);
+
+// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and
+// escape and format it for a tracing line in logging. Absolutely
+// anything including NULs can be in the data. If @scrub is true,
+// non-printing or non-ascii characters are replaced with spaces
+// otherwise a %XX form of escaping is used.
+void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
+ std::string & safe_line);
+
+
#if defined(WIN32)
// Not available on windows where the legacy strtok interface
@@ -78,11 +88,6 @@ namespace LLCore
{
-// ==================================
-// HttpOpRequest
-// ==================================
-
-
HttpOpRequest::HttpOpRequest()
: HttpOperation(),
mProcFlags(0U),
@@ -237,6 +242,19 @@ HttpStatus HttpOpRequest::cancel()
}
+HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id,
+ HttpRequest::priority_t priority,
+ const std::string & url,
+ HttpOptions * options,
+ HttpHeaders * headers)
+{
+ setupCommon(policy_id, priority, url, NULL, options, headers);
+ mReqMethod = HOR_GET;
+
+ return HttpStatus();
+}
+
+
HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
HttpRequest::priority_t priority,
const std::string & url,
@@ -245,30 +263,16 @@ HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
HttpOptions * options,
HttpHeaders * headers)
{
- HttpStatus status;
-
- mProcFlags = 0;
- mReqPolicy = policy_id;
- mReqPriority = priority;
+ setupCommon(policy_id, priority, url, NULL, options, headers);
mReqMethod = HOR_GET;
- mReqURL = url;
mReqOffset = offset;
mReqLength = len;
if (offset || len)
{
mProcFlags |= PF_SCAN_RANGE_HEADER;
}
- if (headers && ! mReqHeaders)
- {
- headers->addRef();
- mReqHeaders = headers;
- }
- if (options && ! mReqOptions)
- {
- mReqOptions = new HttpOptions(*options);
- }
- return status;
+ return HttpStatus();
}
@@ -279,29 +283,10 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id,
HttpOptions * options,
HttpHeaders * headers)
{
- HttpStatus status;
-
- mProcFlags = 0;
- mReqPolicy = policy_id;
- mReqPriority = priority;
+ setupCommon(policy_id, priority, url, body, options, headers);
mReqMethod = HOR_POST;
- mReqURL = url;
- if (body)
- {
- body->addRef();
- mReqBody = body;
- }
- if (headers && ! mReqHeaders)
- {
- headers->addRef();
- mReqHeaders = headers;
- }
- if (options && ! mReqOptions)
- {
- mReqOptions = new HttpOptions(*options);
- }
- return status;
+ return HttpStatus();
}
@@ -312,12 +297,23 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
HttpOptions * options,
HttpHeaders * headers)
{
- HttpStatus status;
+ setupCommon(policy_id, priority, url, body, options, headers);
+ mReqMethod = HOR_PUT;
+
+ return HttpStatus();
+}
+
+void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
+ HttpRequest::priority_t priority,
+ const std::string & url,
+ BufferArray * body,
+ HttpOptions * options,
+ HttpHeaders * headers)
+{
mProcFlags = 0;
mReqPolicy = policy_id;
mReqPriority = priority;
- mReqMethod = HOR_PUT;
mReqURL = url;
if (body)
{
@@ -331,10 +327,14 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
}
if (options && ! mReqOptions)
{
- mReqOptions = new HttpOptions(*options);
+ options->addRef();
+ mReqOptions = options;
+ if (options->getWantHeaders())
+ {
+ mProcFlags |= PF_SAVE_HEADERS;
+ }
+ mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3));
}
-
- return status;
}
@@ -394,30 +394,29 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
const std::string * opt_value(NULL);
- if (policy.get(HttpRequest::GP_CA_PATH, opt_value))
+ long opt_long(0L);
+ policy.get(HttpRequest::GP_LLPROXY, &opt_long);
+ if (opt_long)
{
- curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+ // Use the viewer-based thread-safe API which has a
+ // fast/safe check for proxy enable. Would like to
+ // encapsulate this someway...
+ LLProxy::getInstance()->applyProxySettings(mCurlHandle);
}
- if (policy.get(HttpRequest::GP_CA_FILE, opt_value))
+ else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value))
{
- curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
+ // *TODO: This is fine for now but get fuller socks/
+ // authentication thing going later....
+ curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
+ curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
}
- if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value))
+ if (policy.get(HttpRequest::GP_CA_PATH, &opt_value))
{
- if (*opt_value == "LLProxy")
- {
- // Use the viewer-based thread-safe API which has a
- // fast/safe check for proxy enable. Would like to
- // encapsulate this someway...
- LLProxy::getInstance()->applyProxySettings(mCurlHandle);
- }
- else
- {
- // *TODO: This is fine for now but get fuller socks/
- // authentication thing going later....
- curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
- curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
- }
+ curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+ }
+ if (policy.get(HttpRequest::GP_CA_FILE, &opt_value))
+ {
+ curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
}
switch (mReqMethod)
@@ -463,6 +462,14 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
// *FIXME: fail out here
break;
}
+
+ // Tracing
+ if (mTracing > 1)
+ {
+ curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
+ curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle);
+ curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
+ }
// There's a CURLOPT for this now...
if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
@@ -621,6 +628,101 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
return hdr_size;
}
+
+int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata)
+{
+ HttpOpRequest * op(NULL);
+ curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
+ // *FIXME: check the pointer
+
+ std::string safe_line;
+ std::string tag;
+ bool logit(false);
+ len = (std::min)(len, size_t(256)); // Keep things reasonable in all cases
+
+ switch (info)
+ {
+ case CURLINFO_TEXT:
+ if (op->mTracing > 1)
+ {
+ tag = "TEXT";
+ escape_libcurl_debug_data(buffer, len, true, safe_line);
+ logit = true;
+ }
+ break;
+
+ case CURLINFO_HEADER_IN:
+ if (op->mTracing > 1)
+ {
+ tag = "HEADERIN";
+ escape_libcurl_debug_data(buffer, len, true, safe_line);
+ logit = true;
+ }
+ break;
+
+ case CURLINFO_HEADER_OUT:
+ if (op->mTracing > 1)
+ {
+ tag = "HEADEROUT";
+ escape_libcurl_debug_data(buffer, len, true, safe_line);
+ logit = true;
+ }
+ break;
+
+ case CURLINFO_DATA_IN:
+ if (op->mTracing > 1)
+ {
+ tag = "DATAIN";
+ logit = true;
+ if (op->mTracing > 2)
+ {
+ escape_libcurl_debug_data(buffer, len, false, safe_line);
+ }
+ else
+ {
+ std::ostringstream out;
+ out << len << " Bytes";
+ safe_line = out.str();
+ }
+ }
+ break;
+
+ case CURLINFO_DATA_OUT:
+ if (op->mTracing > 1)
+ {
+ tag = "DATAOUT";
+ logit = true;
+ if (op->mTracing > 2)
+ {
+ escape_libcurl_debug_data(buffer, len, false, safe_line);
+ }
+ else
+ {
+ std::ostringstream out;
+ out << len << " Bytes";
+ safe_line = out.str();
+ }
+ }
+ break;
+
+ default:
+ logit = false;
+ break;
+ }
+
+ if (logit)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle: "
+ << static_cast<HttpHandle>(op)
+ << ", Type: " << tag
+ << ", Data: " << safe_line
+ << LL_ENDL;
+ }
+
+ return 0;
+}
+
+
} // end namespace LLCore
@@ -694,6 +796,43 @@ char *strtok_r(char *str, const char *delim, char ** savestate)
#endif
+
+void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
+{
+ std::string out;
+ len = (std::min)(len, size_t(200));
+ out.reserve(3 * len);
+ for (int i(0); i < len; ++i)
+ {
+ unsigned char uc(static_cast<unsigned char>(buffer[i]));
+
+ if (uc < 32 || uc > 126)
+ {
+ if (scrub)
+ {
+ out.append(1, ' ');
+ }
+ else
+ {
+ static const char hex[] = "0123456789ABCDEF";
+ char convert[4];
+
+ convert[0] = '%';
+ convert[1] = hex[(uc >> 4) % 16];
+ convert[2] = hex[uc % 16];
+ convert[3] = '\0';
+ out.append(convert);
+ }
+ }
+ else
+ {
+ out.append(1, buffer[i]);
+ }
+ }
+ safe_line.swap(out);
+}
+
+
} // end anonymous namespace
diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h
index 80893beb40f38b1ed520bedeb4a8cefc00969c25..fc2301057c94ac07821d3a5e9dc127117274a2c1 100644
--- a/indra/llcorehttp/_httpoprequest.h
+++ b/indra/llcorehttp/_httpoprequest.h
@@ -76,6 +76,12 @@ class HttpOpRequest : public HttpOperation
public:
// Setup Methods
+ HttpStatus setupGet(HttpRequest::policy_t policy_id,
+ HttpRequest::priority_t priority,
+ const std::string & url,
+ HttpOptions * options,
+ HttpHeaders * headers);
+
HttpStatus setupGetByteRange(HttpRequest::policy_t policy_id,
HttpRequest::priority_t priority,
const std::string & url,
@@ -103,15 +109,23 @@ class HttpOpRequest : public HttpOperation
virtual HttpStatus cancel();
protected:
+ void setupCommon(HttpRequest::policy_t policy_id,
+ HttpRequest::priority_t priority,
+ const std::string & url,
+ BufferArray * body,
+ HttpOptions * options,
+ HttpHeaders * headers);
+
static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata);
static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata);
static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata);
+ static int debugCallback(CURL *, curl_infotype info, char * buffer, size_t len, void * userdata);
protected:
unsigned int mProcFlags;
static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U;
static const unsigned int PF_SAVE_HEADERS = 0x00000002U;
-
+
public:
// Request data
EMethod mReqMethod;
diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp
index 21e058b2be56b3aed1e8083f994f53e4bf73567e..c1357f9ae513398438b966d3d1b691de618721fc 100644
--- a/indra/llcorehttp/_httpopsetget.cpp
+++ b/indra/llcorehttp/_httpopsetget.cpp
@@ -89,8 +89,8 @@ void HttpOpSetGet::stageFromRequest(HttpService * service)
}
if (mStatus)
{
- const std::string * value;
- if ((mStatus = pol_opt.get(setting, value)))
+ const std::string * value(NULL);
+ if ((mStatus = pol_opt.get(setting, &value)))
{
mStrValue = *value;
}
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 8ee3f886583f48bb1ca8e51d3b2e96aea0c6c912..0e08d8827605af725c037c9c35be6df6d2e21b4b 100644
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -107,6 +107,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries
<< " being scheduled for " << delta << " uSecs from now."
<< LL_ENDL;
+ if (op->mTracing > 0)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: "
+ << static_cast<HttpHandle>(op)
+ << LL_ENDL;
+ }
mState[policy_class].mRetryQueue.push(op);
}
@@ -224,8 +230,8 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
}
else if (op->mPolicyRetries)
{
- LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries."
- << LL_ENDL;
+ LL_DEBUGS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries."
+ << LL_ENDL;
}
op->stageFromActive(mService);
@@ -234,4 +240,15 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
}
+int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class)
+{
+ if (policy_class < HttpRequest::POLICY_CLASS_LIMIT)
+ {
+ return (mState[policy_class].mReadyQueue.size()
+ + mState[policy_class].mRetryQueue.size());
+ }
+ return 0;
+}
+
+
} // end namespace LLCore
diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h
index 73c22bab789af26830b4ef4e889034ff0c30242d..4114f6484881da4cf1f6f419bca3637b789d1cbc 100644
--- a/indra/llcorehttp/_httppolicy.h
+++ b/indra/llcorehttp/_httppolicy.h
@@ -97,7 +97,10 @@ class HttpPolicy
}
void setPolicies(const HttpPolicyGlobal & global);
-
+
+ // Get ready counts for a particular class
+ int getReadyCount(HttpRequest::policy_t policy_class);
+
protected:
struct State
{
diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp
index d95d73cfba0d717c57d9563d1251f55565501547..6b1de38fd64a5347fd250c9401694f73850832cb 100644
--- a/indra/llcorehttp/_httppolicyglobal.cpp
+++ b/indra/llcorehttp/_httppolicyglobal.cpp
@@ -33,7 +33,9 @@ namespace LLCore
HttpPolicyGlobal::HttpPolicyGlobal()
: mSetMask(0UL),
- mConnectionLimit(32L)
+ mConnectionLimit(32L),
+ mTrace(0),
+ mUseLLProxy(0)
{}
@@ -50,6 +52,8 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other)
mCAPath = other.mCAPath;
mCAFile = other.mCAFile;
mHttpProxy = other.mHttpProxy;
+ mTrace = other.mTrace;
+ mUseLLProxy = other.mUseLLProxy;
}
return *this;
}
@@ -63,6 +67,14 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value)
mConnectionLimit = value;
break;
+ case HttpRequest::GP_TRACE:
+ mTrace = llclamp(value, 0L, 3L);
+ break;
+
+ case HttpRequest::GP_LLPROXY:
+ mUseLLProxy = llclamp(value, 0L, 1L);
+ break;
+
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
@@ -97,54 +109,64 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri
}
-HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long & value)
+HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value)
{
static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
+ long * src(NULL);
switch (opt)
{
case HttpRequest::GP_CONNECTION_LIMIT:
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
- value = mConnectionLimit;
+ src = &mConnectionLimit;
+ break;
+
+ case HttpRequest::GP_TRACE:
+ src = &mTrace;
+ break;
+
+ case HttpRequest::GP_LLPROXY:
+ src = &mUseLLProxy;
break;
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
+ if (! (mSetMask & (1UL << int(opt))))
+ return not_set;
+
+ *value = *src;
return HttpStatus();
}
-HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string *& value)
+HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value)
{
static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
+ const std::string * src(NULL);
switch (opt)
{
case HttpRequest::GP_CA_PATH:
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
- value = &mCAPath;
+ src = &mCAPath;
break;
case HttpRequest::GP_CA_FILE:
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
- value = &mCAFile;
+ src = &mCAFile;
break;
case HttpRequest::GP_HTTP_PROXY:
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
- value = &mHttpProxy;
+ src = &mHttpProxy;
break;
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
+ if (! (mSetMask & (1UL << int(opt))))
+ return not_set;
+
+ *value = src;
return HttpStatus();
}
diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h
index f4bb4d4b258dc4841d05bb08d3f4bf7ecf956b5e..a50d0e41887c4a47f44bc2fcce78fd008484e447 100644
--- a/indra/llcorehttp/_httppolicyglobal.h
+++ b/indra/llcorehttp/_httppolicyglobal.h
@@ -48,8 +48,8 @@ class HttpPolicyGlobal
public:
HttpStatus set(HttpRequest::EGlobalPolicy opt, long value);
HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value);
- HttpStatus get(HttpRequest::EGlobalPolicy opt, long & value);
- HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string *& value);
+ HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value);
+ HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value);
public:
unsigned long mSetMask;
@@ -57,6 +57,8 @@ class HttpPolicyGlobal
std::string mCAPath;
std::string mCAFile;
std::string mHttpProxy;
+ long mTrace;
+ long mUseLLProxy;
}; // end class HttpPolicyGlobal
} // end namespace LLCore
diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp
index 920a3f3b6d029ab41eef8165f872b86e5dcf99b0..beba8f08f4f2d319b29868afd2ab1bbb3521bbcf 100644
--- a/indra/llcorehttp/_httpservice.cpp
+++ b/indra/llcorehttp/_httpservice.cpp
@@ -36,6 +36,7 @@
#include "_thread.h"
#include "lltimer.h"
+#include "llthread.h"
// Tuning parameters
@@ -186,8 +187,10 @@ void HttpService::shutdown()
void HttpService::threadRun(LLCoreInt::HttpThread * thread)
{
boost::this_thread::disable_interruption di;
- ELoopSpeed loop(REQUEST_SLEEP);
+
+ LLThread::registerThreadID();
+ ELoopSpeed loop(REQUEST_SLEEP);
while (! mExitRequested)
{
loop = processRequestQueue(loop);
@@ -226,6 +229,19 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop)
// Process operation
if (! mExitRequested)
{
+ // Setup for subsequent tracing
+ long tracing(0);
+ mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing);
+ op->mTracing = (std::max)(op->mTracing, int(tracing));
+
+ if (op->mTracing > 0)
+ {
+ LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: "
+ << static_cast<HttpHandle>(op)
+ << LL_ENDL;
+ }
+
+ // Stage
op->stageFromRequest(this);
}
diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp
index 15b505f5bfd1e9cb3b63de676d6612ebab58e318..155fbda7f1d752429ecb2992c8dca596a4759b01 100644
--- a/indra/llcorehttp/httpoptions.cpp
+++ b/indra/llcorehttp/httpoptions.cpp
@@ -33,13 +33,15 @@ namespace LLCore
HttpOptions::HttpOptions()
: RefCounted(true),
- mWantHeaders(false)
+ mWantHeaders(false),
+ mTracing(0)
{}
HttpOptions::HttpOptions(const HttpOptions & rhs)
: RefCounted(true),
- mWantHeaders(rhs.mWantHeaders)
+ mWantHeaders(rhs.mWantHeaders),
+ mTracing(rhs.mTracing)
{}
@@ -47,4 +49,16 @@ HttpOptions::~HttpOptions()
{}
+void HttpOptions::setWantHeaders()
+{
+ mWantHeaders = true;
+}
+
+
+void HttpOptions::setTrace(long level)
+{
+ mTracing = int(level);
+}
+
+
} // end namespace LLCore
diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h
index 267a982dd5e5c165afc7f9b8c9b11d032dba3bb0..0b9dfdc1de2297a160e21b70a71e3bf727e753f0 100644
--- a/indra/llcorehttp/httpoptions.h
+++ b/indra/llcorehttp/httpoptions.h
@@ -67,11 +67,21 @@ class HttpOptions : public LLCoreInt::RefCounted
void operator=(const HttpOptions &); // Not defined
public:
+ void setWantHeaders();
+ bool getWantHeaders() const
+ {
+ return mWantHeaders;
+ }
+
+ void setTrace(long level);
+ int getTrace() const
+ {
+ return mTracing;
+ }
protected:
- // *TODO: add some options
bool mWantHeaders;
-
+ long int mTracing;
}; // end class HttpOptions
diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp
index 089eee76f3315a79ee628a6ebd9b48ae3f96b302..2036ecfd1c4c6c1b492a2e87f5e10357829ce7e3 100644
--- a/indra/llcorehttp/httprequest.cpp
+++ b/indra/llcorehttp/httprequest.cpp
@@ -135,6 +135,33 @@ HttpStatus HttpRequest::getStatus() const
}
+HttpHandle HttpRequest::requestGet(policy_t policy_id,
+ priority_t priority,
+ const std::string & url,
+ HttpOptions * options,
+ HttpHeaders * headers,
+ HttpHandler * user_handler)
+{
+ HttpStatus status;
+ HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID);
+
+ HttpOpRequest * op = new HttpOpRequest();
+ if (! (status = op->setupGet(policy_id, priority, url, options, headers)))
+ {
+ op->release();
+ mLastReqStatus = status;
+ return handle;
+ }
+ op->setReplyPath(mReplyQueue, user_handler);
+ mRequestQueue->addOp(op); // transfers refcount
+
+ mLastReqStatus = status;
+ handle = static_cast<HttpHandle>(op);
+
+ return handle;
+}
+
+
HttpHandle HttpRequest::requestGetByteRange(policy_t policy_id,
priority_t priority,
const std::string & url,
diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h
index a953aa28d0a799c6f84d539477db113b337cf010..4e78ed37190c16ed034274f1213b0e9f3081cea2 100644
--- a/indra/llcorehttp/httprequest.h
+++ b/indra/llcorehttp/httprequest.h
@@ -111,10 +111,43 @@ class HttpRequest
/// Maximum number of connections the library will use to
/// perform operations. This is somewhat soft as the underlying
/// transport will cache some connections (up to 5).
- GP_CONNECTION_LIMIT, ///< Takes long giving number of connections
- GP_CA_PATH, ///< System path/directory where SSL certs are stored.
- GP_CA_FILE, ///< System path/file containing certs.
- GP_HTTP_PROXY ///< String giving host/port to use for HTTP proxy
+
+ /// A long value setting the maximum number of connections
+ /// allowed over all policy classes. Note that this will be
+ /// a somewhat soft value. There may be an additional five
+ /// connections per policy class depending upon runtime
+ /// behavior.
+ GP_CONNECTION_LIMIT,
+
+ /// String containing a system-appropriate directory name
+ /// where SSL certs are stored.
+ GP_CA_PATH,
+
+ /// String giving a full path to a file containing SSL certs.
+ GP_CA_FILE,
+
+ /// String of host/port to use as simple HTTP proxy. This is
+ /// going to change in the future into something more elaborate
+ /// that may support richer schemes.
+ GP_HTTP_PROXY,
+
+ /// Long value that if non-zero enables the use of the
+ /// traditional LLProxy code for http/socks5 support. If
+ /// enabled, has priority over GP_HTTP_PROXY.
+ GP_LLPROXY,
+
+ /// Long value setting the logging trace level for the
+ /// library. Possible values are:
+ /// 0 - No tracing (default)
+ /// 1 - Basic tracing of request start, stop and major events.
+ /// 2 - Connection, header and payload size information from
+ /// HTTP transactions.
+ /// 3 - Partial logging of payload itself.
+ ///
+ /// These values are also used in the trace modes for
+ /// individual requests in HttpOptions. Also be aware that
+ /// tracing tends to impact performance of the viewer.
+ GP_TRACE
};
/// Set a parameter on a global policy option. Calls
@@ -133,7 +166,7 @@ class HttpRequest
/// the class in other methods. If -1, an error
/// occurred and @see getStatus() may provide more
/// detail on the reason.
- policy_t createPolicyClass();
+ static policy_t createPolicyClass();
enum EClassPolicy
{
@@ -157,7 +190,7 @@ class HttpRequest
/// @param opt Enum of option to be set.
/// @param value Desired value of option.
/// @return Standard status code.
- HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
+ static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
/// @}
@@ -176,6 +209,36 @@ class HttpRequest
///
HttpStatus getStatus() const;
+ /// Queue a full HTTP GET request to be issued for entire entity.
+ /// The request is queued and serviced by the working thread and
+ /// notification of completion delivered to the optional HttpHandler
+ /// argument during @see update() calls.
+ ///
+ /// With a valid handle returned, it can be used to reference the
+ /// request in other requests (like cancellation) and will be an
+ /// argument when any HttpHandler object is invoked.
+ ///
+ /// @param policy_id Default or user-defined policy class under
+ /// which this request is to be serviced.
+ /// @param priority Standard priority scheme inherited from
+ /// Indra code base (U32-type scheme).
+ /// @param url
+ /// @param options (optional)
+ /// @param headers (optional)
+ /// @param handler (optional)
+ /// @return The handle of the request if successfully
+ /// queued or LLCORE_HTTP_HANDLE_INVALID if the
+ /// request could not be queued. In the latter
+ /// case, @see getStatus() will return more info.
+ ///
+ HttpHandle requestGet(policy_t policy_id,
+ priority_t priority,
+ const std::string & url,
+ HttpOptions * options,
+ HttpHeaders * headers,
+ HttpHandler * handler);
+
+
/// Queue a full HTTP GET request to be issued with a 'Range' header.
/// The request is queued and serviced by the working thread and
/// notification of completion delivered to the optional HttpHandler
diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp
index 61698f34d84c89754f0de50d126bee0ac1530325..5b04796c8ad9c454cbd004be1cc57c26199caebe 100644
--- a/indra/llcorehttp/tests/test_httprequest.hpp
+++ b/indra/llcorehttp/tests/test_httprequest.hpp
@@ -395,7 +395,7 @@ void HttpRequestTestObjectType::test<5>()
{
ScopedCurlInit ready;
- set_test_name("HttpRequest GET + Stop execution");
+ set_test_name("HttpRequest GET to dead port + Stop execution");
// Handler can be stack-allocated *if* there are no dangling
// references to it after completion of this method.
@@ -496,6 +496,7 @@ void HttpRequestTestObjectType::test<5>()
}
}
+
template <> template <>
void HttpRequestTestObjectType::test<6>()
{
@@ -517,6 +518,114 @@ void HttpRequestTestObjectType::test<6>()
HttpRequest * req = NULL;
+ try
+ {
+ // Get singletons created
+ HttpRequest::createService();
+
+ // Start threading early so that thread memory is invariant
+ // over the test.
+ HttpRequest::startThread();
+
+ // create a new ref counted object with an implicit reference
+ req = new HttpRequest();
+ ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
+
+ // Issue a GET that *can* connect
+ mStatus = HttpStatus(200);
+ HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID,
+ 0U,
+ url_base,
+ NULL,
+ NULL,
+ &handler);
+ ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump.
+ int count(0);
+ int limit(10);
+ while (count++ < limit && mHandlerCalls < 1)
+ {
+ req->update(1000);
+ usleep(100000);
+ }
+ ensure("Request executed in reasonable time", count < limit);
+ ensure("One handler invocation for request", mHandlerCalls == 1);
+
+ // Okay, request a shutdown of the servicing thread
+ mStatus = HttpStatus();
+ handle = req->requestStopThread(&handler);
+ ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump again
+ count = 0;
+ limit = 10;
+ while (count++ < limit && mHandlerCalls < 2)
+ {
+ req->update(1000);
+ usleep(100000);
+ }
+ ensure("Second request executed in reasonable time", count < limit);
+ ensure("Second handler invocation", mHandlerCalls == 2);
+
+ // See that we actually shutdown the thread
+ count = 0;
+ limit = 10;
+ while (count++ < limit && ! HttpService::isStopped())
+ {
+ usleep(100000);
+ }
+ ensure("Thread actually stopped running", HttpService::isStopped());
+
+ // release the request object
+ delete req;
+ req = NULL;
+
+ // Shut down service
+ HttpRequest::destroyService();
+
+ ensure("Two handler calls on the way out", 2 == mHandlerCalls);
+
+#if defined(WIN32)
+ // Can only do this memory test on Windows. On other platforms,
+ // the LL logging system holds on to memory and produces what looks
+ // like memory leaks...
+
+ // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal());
+ ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal());
+#endif
+ }
+ catch (...)
+ {
+ stop_thread(req);
+ delete req;
+ HttpRequest::destroyService();
+ throw;
+ }
+}
+
+
+template <> template <>
+void HttpRequestTestObjectType::test<7>()
+{
+ ScopedCurlInit ready;
+
+ std::string url_base(get_base_url());
+ std::cerr << "Base: " << url_base << std::endl;
+
+ set_test_name("HttpRequest GET with Range: header to real service");
+
+ // Handler can be stack-allocated *if* there are no dangling
+ // references to it after completion of this method.
+ // Create before memory record as the string copy will bump numbers.
+ TestHandler2 handler(this, "handler");
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+ mHandlerCalls = 0;
+
+ HttpRequest * req = NULL;
+
try
{
// Get singletons created
@@ -605,8 +714,9 @@ void HttpRequestTestObjectType::test<6>()
}
}
+
template <> template <>
-void HttpRequestTestObjectType::test<7>()
+void HttpRequestTestObjectType::test<8>()
{
ScopedCurlInit ready;
@@ -725,7 +835,7 @@ void HttpRequestTestObjectType::test<7>()
}
template <> template <>
-void HttpRequestTestObjectType::test<8>()
+void HttpRequestTestObjectType::test<9>()
{
ScopedCurlInit ready;
@@ -843,6 +953,118 @@ void HttpRequestTestObjectType::test<8>()
}
}
+template <> template <>
+void HttpRequestTestObjectType::test<10>()
+{
+ ScopedCurlInit ready;
+
+ std::string url_base(get_base_url());
+ std::cerr << "Base: " << url_base << std::endl;
+
+ set_test_name("HttpRequest GET with some tracing");
+
+ // Handler can be stack-allocated *if* there are no dangling
+ // references to it after completion of this method.
+ // Create before memory record as the string copy will bump numbers.
+ TestHandler2 handler(this, "handler");
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+ mHandlerCalls = 0;
+
+ HttpRequest * req = NULL;
+
+ try
+ {
+ // Get singletons created
+ HttpRequest::createService();
+
+ // Enable tracing
+ HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2);
+
+ // Start threading early so that thread memory is invariant
+ // over the test.
+ HttpRequest::startThread();
+
+ // create a new ref counted object with an implicit reference
+ req = new HttpRequest();
+ ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
+
+ // Issue a GET that *can* connect
+ mStatus = HttpStatus(200);
+ HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
+ 0U,
+ url_base,
+ 0,
+ 0,
+ NULL,
+ NULL,
+ &handler);
+ ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump.
+ int count(0);
+ int limit(10);
+ while (count++ < limit && mHandlerCalls < 1)
+ {
+ req->update(1000);
+ usleep(100000);
+ }
+ ensure("Request executed in reasonable time", count < limit);
+ ensure("One handler invocation for request", mHandlerCalls == 1);
+
+ // Okay, request a shutdown of the servicing thread
+ mStatus = HttpStatus();
+ handle = req->requestStopThread(&handler);
+ ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump again
+ count = 0;
+ limit = 10;
+ while (count++ < limit && mHandlerCalls < 2)
+ {
+ req->update(1000);
+ usleep(100000);
+ }
+ ensure("Second request executed in reasonable time", count < limit);
+ ensure("Second handler invocation", mHandlerCalls == 2);
+
+ // See that we actually shutdown the thread
+ count = 0;
+ limit = 10;
+ while (count++ < limit && ! HttpService::isStopped())
+ {
+ usleep(100000);
+ }
+ ensure("Thread actually stopped running", HttpService::isStopped());
+
+ // release the request object
+ delete req;
+ req = NULL;
+
+ // Shut down service
+ HttpRequest::destroyService();
+
+ ensure("Two handler calls on the way out", 2 == mHandlerCalls);
+
+#if defined(WIN32)
+ // Can only do this memory test on Windows. On other platforms,
+ // the LL logging system holds on to memory and produces what looks
+ // like memory leaks...
+
+ // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal());
+ ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal());
+#endif
+ }
+ catch (...)
+ {
+ stop_thread(req);
+ delete req;
+ HttpRequest::destroyService();
+ throw;
+ }
+}
+
} // end namespace tut
namespace
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index e2c13e77e3a4f2cbfd1922f2940c4d1a545034a1..430dd89c3e497e637a461875b57eb9e74652f89d 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -5356,11 +5356,9 @@ void CoreHttp::init()
<< LL_ENDL;
}
- mRequest = new LLCore::HttpRequest;
-
// Point to our certs or SSH/https: will fail on connect
- status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE,
- gDirUtilp->getCAFile());
+ status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE,
+ gDirUtilp->getCAFile());
if (! status)
{
LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: "
@@ -5371,15 +5369,22 @@ void CoreHttp::init()
// Establish HTTP Proxy. "LLProxy" is a special string which directs
// the code to use LLProxy::applyProxySettings() to establish any
// HTTP or SOCKS proxy for http operations.
- status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_HTTP_PROXY,
- std::string("LLProxy"));
+ status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1);
if (! status)
{
LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: "
<< status.toString()
<< LL_ENDL;
}
-
+
+ // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy):
+ // 0 - None
+ // 1 - Basic start, stop simple transitions
+ // 2 - libcurl CURLOPT_VERBOSE mode with brief lines
+ // 3 - with partial data content
+ status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0);
+
+ // Kick the thread
status = LLCore::HttpRequest::startThread();
if (! status)
{
@@ -5388,6 +5393,7 @@ void CoreHttp::init()
<< LL_ENDL;
}
+ mRequest = new LLCore::HttpRequest;
}
diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp
index f5e7540e85e432d7c96493784c91f9f8d925fea6..664af02f78c26f3f0911311a9c3285413e9d292e 100644
--- a/indra/newview/lltexturefetch.cpp
+++ b/indra/newview/lltexturefetch.cpp
@@ -431,6 +431,22 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler
void lockWorkMutex() { mWorkMutex.lock(); }
void unlockWorkMutex() { mWorkMutex.unlock(); }
+ // Locks: Mw
+ void acquireHttpSemaphore()
+ {
+ llassert(! mHttpHasResource);
+ mHttpHasResource = true;
+ --mFetcher->mHttpSemaphore;
+ }
+
+ // Locks: Mw
+ void releaseHttpSemaphore()
+ {
+ llassert(mHttpHasResource);
+ mHttpHasResource = false;
+ ++mFetcher->mHttpSemaphore;
+ }
+
private:
enum e_state // mState
{
@@ -444,8 +460,9 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler
CACHE_POST,
LOAD_FROM_NETWORK,
LOAD_FROM_SIMULATOR,
- SEND_HTTP_REQ, // Commit to sending as HTTP
WAIT_HTTP_RESOURCE, // Waiting for HTTP resources
+ WAIT_HTTP_RESOURCE2, // Waiting for HTTP resources
+ SEND_HTTP_REQ, // Commit to sending as HTTP
WAIT_HTTP_REQ, // Request sent, wait for completion
DECODE_IMAGE,
DECODE_IMAGE_UPDATE,
@@ -532,7 +549,7 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler
bool mHttpActive; // Active request to http library
unsigned int mHttpReplySize;
unsigned int mHttpReplyOffset;
- bool mHttpReleased; // Has been released from resource wait once
+ bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore
};
//////////////////////////////////////////////////////////////////////////////
@@ -768,8 +785,9 @@ const char* LLTextureFetchWorker::sStateDescs[] = {
"CACHE_POST",
"LOAD_FROM_NETWORK",
"LOAD_FROM_SIMULATOR",
- "SEND_HTTP_REQ",
"WAIT_HTTP_RESOURCE",
+ "WAIT_HTTP_RESOURCE2",
+ "SEND_HTTP_REQ",
"WAIT_HTTP_REQ",
"DECODE_IMAGE",
"DECODE_IMAGE_UPDATE",
@@ -836,7 +854,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher,
mHttpActive(false),
mHttpReplySize(0U),
mHttpReplyOffset(0U),
- mHttpReleased(true)
+ mHttpHasResource(false)
{
mCanUseNET = mUrl.empty() ;
@@ -860,6 +878,10 @@ LLTextureFetchWorker::~LLTextureFetchWorker()
llassert_always(!haveWork());
lockWorkMutex(); // +Mw (should be useless)
+ if (mHttpHasResource)
+ {
+ releaseHttpSemaphore();
+ }
if (mHttpActive)
{
// Issue a cancel on a live request...
@@ -1126,7 +1148,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
llwarns << "Unknown URL Type: " << mUrl << llendl;
}
setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
- mState = SEND_HTTP_REQ;
+ mState = WAIT_HTTP_RESOURCE;
}
else
{
@@ -1223,7 +1245,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
}
if (mCanUseHTTP && !mUrl.empty())
{
- mState = SEND_HTTP_REQ;
+ mState = WAIT_HTTP_RESOURCE;
setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
if(mWriteToCacheState != NOT_WRITE)
{
@@ -1287,31 +1309,38 @@ bool LLTextureFetchWorker::doWork(S32 param)
return false;
}
- if (mState == SEND_HTTP_REQ)
+ if (mState == WAIT_HTTP_RESOURCE)
{
- if (! mCanUseHTTP)
- {
- return true; // abort
- }
-
// NOTE:
// control the number of the http requests issued for:
// 1, not openning too many file descriptors at the same time;
// 2, control the traffic of http so udp gets bandwidth.
//
- if (! mHttpReleased)
+ // If it looks like we're busy, keep this request here.
+ // Otherwise, advance into the HTTP states.
+ if (mFetcher->mHttpSemaphore <= 0 || mFetcher->getHttpWaitersCount())
{
- // If this request hasn't been released before and it looks like
- // we're busy, put this request into resource wait and allow something
- // else to come to the front.
- if (mFetcher->getNumHTTPRequests() >= HTTP_REQUESTS_IN_QUEUE_HIGH_WATER ||
- mFetcher->getHttpWaitersCount())
- {
- mState = WAIT_HTTP_RESOURCE;
- setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
- mFetcher->addHttpWaiter(this->mID);
- return false;
- }
+ mState = WAIT_HTTP_RESOURCE2;
+ setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
+ mFetcher->addHttpWaiter(this->mID);
+ return false;
+ }
+ mState = SEND_HTTP_REQ;
+ acquireHttpSemaphore();
+ }
+
+ if (mState == WAIT_HTTP_RESOURCE2)
+ {
+ // Just idle it if we make it to the head...
+ return false;
+ }
+
+ if (mState == SEND_HTTP_REQ)
+ {
+ if (! mCanUseHTTP)
+ {
+ releaseHttpSemaphore();
+ return true; // abort
}
mFetcher->removeFromNetworkQueue(this, false);
@@ -1327,10 +1356,12 @@ bool LLTextureFetchWorker::doWork(S32 param)
// We already have all the data, just decode it
mLoadedDiscard = mFormattedImage->getDiscardLevel();
mState = DECODE_IMAGE;
+ releaseHttpSemaphore();
return false;
}
else
{
+ releaseHttpSemaphore();
return true; // abort.
}
}
@@ -1365,6 +1396,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
{
llwarns << "HTTP GET request failed for " << mID << llendl;
resetFormattedData();
+ releaseHttpSemaphore();
return true; // failed
}
@@ -1377,13 +1409,6 @@ bool LLTextureFetchWorker::doWork(S32 param)
// fall through
}
- if (mState == WAIT_HTTP_RESOURCE)
- {
- // Nothing to do until releaseHttpWaiters() puts us back
- // into the flow...
- return false;
- }
-
if (mState == WAIT_HTTP_REQ)
{
if (mLoaded)
@@ -1401,6 +1426,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
mState = INIT;
mCanUseHTTP = false;
setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
+ releaseHttpSemaphore();
return false;
}
}
@@ -1423,12 +1449,14 @@ bool LLTextureFetchWorker::doWork(S32 param)
// Use available data
mLoadedDiscard = mFormattedImage->getDiscardLevel();
mState = DECODE_IMAGE;
+ releaseHttpSemaphore();
return false;
}
// Fail harder
resetFormattedData();
mState = DONE;
+ releaseHttpSemaphore();
return true; // failed
}
@@ -1443,6 +1471,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
// abort.
mState = DONE;
+ releaseHttpSemaphore();
return true;
}
@@ -1491,6 +1520,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
mWriteToCacheState = SHOULD_WRITE ;
}
setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
+ releaseHttpSemaphore();
return false;
}
else
@@ -2137,7 +2167,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image
mQAMode(qa_mode),
mHttpRequest(NULL),
mHttpOptions(NULL),
- mHttpHeaders(NULL)
+ mHttpHeaders(NULL),
+ mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER)
{
mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS");
mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold"));
@@ -3155,12 +3186,11 @@ void LLTextureFetch::removeHttpWaiter(const LLUUID & tid)
// Locks: -Mw (must not hold any worker when called)
void LLTextureFetch::releaseHttpWaiters()
{
- if (HTTP_REQUESTS_IN_QUEUE_LOW_WATER < getNumHTTPRequests())
+ if (mHttpSemaphore < HTTP_REQUESTS_IN_QUEUE_LOW_WATER)
return;
// Quickly make a copy of all the LLUIDs. Get off the
// mutex as early as possible.
-
typedef std::vector<LLUUID> uuid_vec_t;
uuid_vec_t tids;
@@ -3171,13 +3201,12 @@ void LLTextureFetch::releaseHttpWaiters()
return;
const size_t limit(mHttpWaitResource.size());
- tids.resize(limit);
- wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin());
- for (int i(0);
- i < limit && mHttpWaitResource.end() != iter;
- ++i, ++iter)
+ tids.reserve(limit);
+ for (wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin());
+ mHttpWaitResource.end() != iter;
+ ++iter)
{
- tids[i] = *iter;
+ tids.push_back(*iter);
}
} // -Mfnq
@@ -3196,28 +3225,29 @@ void LLTextureFetch::releaseHttpWaiters()
tids2.insert(worker);
}
}
+ tids.clear();
// Release workers up to the high water mark. Since we aren't
// holding any locks at this point, we can be in competition
// with other callers. Do defensive things like getting
// refreshed counts of requests and checking if someone else
// has moved any worker state around....
- tids.clear();
for (worker_set_t::iterator iter2(tids2.begin());
- tids2.end() != iter2 && 0 < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - getNumHTTPRequests());
+ tids2.end() != iter2 && mHttpSemaphore > 0;
++iter2)
{
LLTextureFetchWorker * worker(* iter2);
worker->lockWorkMutex(); // +Mw
- if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE != worker->mState)
+ if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE2 != worker->mState)
{
worker->unlockWorkMutex(); // -Mw
continue;
}
- worker->mHttpReleased = true;
+
worker->mState = LLTextureFetchWorker::SEND_HTTP_REQ;
worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority);
+ worker->acquireHttpSemaphore();
worker->unlockWorkMutex(); // -Mw
removeHttpWaiter(worker->mID);
@@ -3456,7 +3486,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher)
}
// In QA mode, Metrics submode, log the result for ease of testing
- if (fetcher->isQAMode() || true)
+ if (fetcher->isQAMode())
{
LL_INFOS("Textures") << merged_llsd << LL_ENDL;
}
diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h
index 4ee13d171e843fbbf06500eb445f2f26db87ac24..50e318162363378775cb218063515531da85840c 100644
--- a/indra/newview/lltexturefetch.h
+++ b/indra/newview/lltexturefetch.h
@@ -312,6 +312,15 @@ class LLTextureFetch : public LLWorkerThread
LLCore::HttpOptions * mHttpOptions; // Ttf
LLCore::HttpHeaders * mHttpHeaders; // Ttf
+ // We use a resource semaphore to keep HTTP requests in
+ // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the
+ // transport. This keeps them near where they can be cheaply
+ // reprioritized rather than dumping them all across a thread
+ // where it's more expensive to get at them. Requests in either
+ // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore
+ // and tracking state transitions is critical to liveness.
+ int mHttpSemaphore; // Ttf
+
typedef std::set<LLUUID> wait_http_res_queue_t;
wait_http_res_queue_t mHttpWaitResource; // Mfnq
diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp
index 5f1d7829ed16e6c6dc45d39543450c4d93c3971f..bb1535d23db7767c504142a95d1a31aadce41373 100644
--- a/indra/newview/lltextureview.cpp
+++ b/indra/newview/lltextureview.cpp
@@ -234,15 +234,16 @@ void LLTextureBar::draw()
{ "DSK", LLColor4::blue }, // CACHE_POST
{ "NET", LLColor4::green }, // LOAD_FROM_NETWORK
{ "SIM", LLColor4::green }, // LOAD_FROM_SIMULATOR
+ { "HTW", LLColor4::green }, // WAIT_HTTP_RESOURCE
+ { "HTW", LLColor4::green }, // WAIT_HTTP_RESOURCE2
{ "REQ", LLColor4::yellow },// SEND_HTTP_REQ
- { "HTW", LLColor4::green }, // WAIT_HTTP_RES
{ "HTP", LLColor4::green }, // WAIT_HTTP_REQ
{ "DEC", LLColor4::yellow },// DECODE_IMAGE
{ "DEC", LLColor4::green }, // DECODE_IMAGE_UPDATE
{ "WRT", LLColor4::purple },// WRITE_TO_CACHE
{ "WRT", LLColor4::orange },// WAIT_ON_WRITE
{ "END", LLColor4::red }, // DONE
-#define LAST_STATE 13
+#define LAST_STATE 14
{ "CRE", LLColor4::magenta }, // LAST_STATE+1
{ "FUL", LLColor4::green }, // LAST_STATE+2
{ "BAD", LLColor4::red }, // LAST_STATE+3