From 05af16a23abe37210e0b880aa27387d8994419dd Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Wed, 6 Jun 2012 13:52:38 -0400
Subject: [PATCH] Policy + caching fixes + https support + POST working
Implemented first global policy definitions to support SSL CA certificate
configuration to support https: operations. Fixed HTTP 206 status handling
to match what is currently being done by grid services and to lay a
foundation for fixes that will be a response to ER-1824. More libcurl
CURLOPT options set on easy handles to do peer verification in the
traditional way. HTTP POST working and now reporting asset metrics back to
grid for the viewer's asset system. This uses LLSD so that is also showing
as compatible with the new library.
---
indra/llcorehttp/CMakeLists.txt | 2 ++
indra/llcorehttp/_httplibcurl.cpp | 1 +
indra/llcorehttp/_httpoprequest.cpp | 45 +++++++++++++++++++++++++----
indra/llcorehttp/_httppolicy.cpp | 4 +--
indra/llcorehttp/_httppolicy.h | 10 +++++++
indra/llcorehttp/_httpservice.h | 8 ++---
indra/llcorehttp/httpcommon.cpp | 5 +++-
indra/llcorehttp/httpcommon.h | 11 ++++++-
indra/llcorehttp/httprequest.cpp | 13 +++++++--
indra/llcorehttp/httprequest.h | 16 +++++-----
indra/newview/llappviewer.cpp | 12 +++++++-
indra/newview/lltexturefetch.cpp | 37 ++++++++++++++----------
12 files changed, 126 insertions(+), 38 deletions(-)
diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt
index 9a073eb8501..3fda524ddfb 100644
--- a/indra/llcorehttp/CMakeLists.txt
+++ b/indra/llcorehttp/CMakeLists.txt
@@ -33,6 +33,7 @@ set(llcorehttp_SOURCE_FILES
_httpoprequest.cpp
_httpopsetpriority.cpp
_httppolicy.cpp
+ _httppolicyglobal.cpp
_httpreplyqueue.cpp
_httprequestqueue.cpp
_httpservice.cpp
@@ -55,6 +56,7 @@ set(llcorehttp_HEADER_FILES
_httpoprequest.h
_httpopsetpriority.h
_httppolicy.h
+ _httppolicyglobal.h
_httpreadyqueue.h
_httpreplyqueue.h
_httprequestqueue.h
diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index 704f9baac91..5272c391e8e 100644
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -27,6 +27,7 @@
#include "_httplibcurl.h"
#include "httpheaders.h"
+#include "bufferarray.h"
#include "_httpoprequest.h"
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index f52ff5a44c4..4bdc4a5257c 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -40,8 +40,10 @@
#include "_httpreplyqueue.h"
#include "_httpservice.h"
#include "_httppolicy.h"
+#include "_httppolicyglobal.h"
#include "_httplibcurl.h"
+#include "llhttpstatuscodes.h"
namespace
{
@@ -153,14 +155,14 @@ HttpOpRequest::~HttpOpRequest()
void HttpOpRequest::stageFromRequest(HttpService * service)
{
addRef();
- service->getPolicy()->addOp(this); // transfers refcount
+ service->getPolicy().addOp(this); // transfers refcount
}
void HttpOpRequest::stageFromReady(HttpService * service)
{
addRef();
- service->getTransport()->addOp(this); // transfers refcount
+ service->getTransport().addOp(this); // transfers refcount
}
@@ -195,6 +197,8 @@ void HttpOpRequest::stageFromActive(HttpService * service)
void HttpOpRequest::visitNotifier(HttpRequest * request)
{
+ static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS);
+
if (mLibraryHandler)
{
HttpResponse * response = new HttpResponse();
@@ -208,9 +212,15 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
offset = mReplyOffset;
length = mReplyLength;
}
- else if (mReplyBody)
+ else if (mReplyBody && partial_content == mStatus)
{
- // Provide implicit offset/length from request/response
+ // Legacy grid services did not provide a 'Content-Range'
+ // header in responses to full- or partly-satisfyiable
+ // 'Range' requests. For these, we have to hope that
+ // the data starts where requested and the length is simply
+ // whatever we received. A bit of sanity could be provided
+ // by overlapping ranged requests and verifying that the
+ // overlap matches.
offset = mReqOffset;
length = mReplyBody->size();
}
@@ -306,6 +316,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
// *FIXME: better error handling later
HttpStatus status;
+ // Get policy options
+ HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions());
+
mCurlHandle = curl_easy_init();
// curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30);
@@ -322,21 +335,40 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
- curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10);
+ curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); // *FIXME: parameterize this later
curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle);
curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle);
+ curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1);
+ curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
+ std::string opt_value;
+ if (policy.get(HttpRequest::GP_CA_PATH, opt_value))
+ {
+ curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value.c_str());
+ }
+ if (policy.get(HttpRequest::GP_CA_FILE, opt_value))
+ {
+ curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value.c_str());
+ }
+ if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value))
+ {
+ curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value.c_str());
+ }
+
switch (mReqMethod)
{
case HOR_GET:
curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
+ mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
+ mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
break;
case HOR_POST:
{
curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
+ curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
long data_size(0);
if (mReqBody)
{
@@ -358,8 +390,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
data_size = mReqBody->size();
}
curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
+ curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
+ mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
+ mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
}
break;
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 1d28f23d563..51f5e487dc8 100644
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -76,12 +76,12 @@ void HttpPolicy::addOp(HttpOpRequest * op)
HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
{
HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP);
- HttpLibcurl * pTransport(mService->getTransport());
+ HttpLibcurl & transport(mService->getTransport());
for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class)
{
HttpReadyQueue & readyq(mReadyQueue[policy_class]);
- int active(pTransport->getActiveCountInClass(policy_class));
+ int active(transport.getActiveCountInClass(policy_class));
int needed(8 - active);
if (needed > 0 && mReadyInClass[policy_class] > 0)
diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h
index 2bc03c531fb..425079ec637 100644
--- a/indra/llcorehttp/_httppolicy.h
+++ b/indra/llcorehttp/_httppolicy.h
@@ -31,6 +31,7 @@
#include "httprequest.h"
#include "_httpservice.h"
#include "_httpreadyqueue.h"
+#include "_httppolicyglobal.h"
namespace LLCore
@@ -68,11 +69,20 @@ class HttpPolicy
// Shadows HttpService's method
bool changePriority(HttpHandle handle, HttpRequest::priority_t priority);
+
+ // Get pointer to global policy options. Caller is expected
+ // to do context checks like no setting once running.
+ HttpPolicyGlobal & getGlobalOptions()
+ {
+ return mGlobalOptions;
+ }
+
protected:
int mReadyInClass[HttpRequest::POLICY_CLASS_LIMIT];
HttpReadyQueue mReadyQueue[HttpRequest::POLICY_CLASS_LIMIT];
HttpService * mService; // Naked pointer, not refcounted, not owner
+ HttpPolicyGlobal mGlobalOptions;
}; // end class HttpPolicy
diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h
index 095316c8a77..748354a8e49 100644
--- a/indra/llcorehttp/_httpservice.h
+++ b/indra/llcorehttp/_httpservice.h
@@ -148,14 +148,14 @@ class HttpService
/// Threading: callable by worker thread.
bool changePriority(HttpHandle handle, HttpRequest::priority_t priority);
- HttpPolicy * getPolicy()
+ HttpPolicy & getPolicy()
{
- return mPolicy;
+ return *mPolicy;
}
- HttpLibcurl * getTransport()
+ HttpLibcurl & getTransport()
{
- return mTransport;
+ return *mTransport;
}
protected:
diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp
index a01182cf231..9f17b5c8426 100644
--- a/indra/llcorehttp/httpcommon.cpp
+++ b/indra/llcorehttp/httpcommon.cpp
@@ -66,7 +66,10 @@ std::string HttpStatus::toString() const
"Services shutting down",
"Operation canceled",
"Invalid Content-Range header encountered",
- "Request handle not found"
+ "Request handle not found",
+ "Invalid datatype for argument or option",
+ "Option has not been explicitly set",
+ "Option is not dynamic and must be set early"
};
static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0]));
diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h
index c01a5f85d30..fd2661b700c 100644
--- a/indra/llcorehttp/httpcommon.h
+++ b/indra/llcorehttp/httpcommon.h
@@ -137,7 +137,16 @@ enum HttpError
HE_INV_CONTENT_RANGE_HDR = 4,
// Request handle not found
- HE_HANDLE_NOT_FOUND = 5
+ HE_HANDLE_NOT_FOUND = 5,
+
+ // Invalid datatype for option/setting
+ HE_INVALID_ARG = 6,
+
+ // Option hasn't been explicitly set
+ HE_OPT_NOT_SET = 7,
+
+ // Option not dynamic, must be set during init phase
+ HE_OPT_NOT_DYNAMIC = 8
}; // end enum HttpError
diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp
index 0e512d97ed5..baa0fe1a842 100644
--- a/indra/llcorehttp/httprequest.cpp
+++ b/indra/llcorehttp/httprequest.cpp
@@ -29,6 +29,7 @@
#include "_httprequestqueue.h"
#include "_httpreplyqueue.h"
#include "_httpservice.h"
+#include "_httppolicy.h"
#include "_httpoperation.h"
#include "_httpoprequest.h"
#include "_httpopsetpriority.h"
@@ -127,9 +128,17 @@ HttpRequest::~HttpRequest()
HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value)
{
- HttpStatus status;
+ // *FIXME: Fail if thread is running.
- return status;
+ return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value);
+}
+
+
+HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value)
+{
+ // *FIXME: Fail if thread is running.
+
+ return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value);
}
diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h
index 57d2da245b6..3592d5c6a37 100644
--- a/indra/llcorehttp/httprequest.h
+++ b/indra/llcorehttp/httprequest.h
@@ -111,7 +111,10 @@ class HttpRequest
/// Maximum number of connections the library will use to
/// perform operations. This is somewhat soft as the underlying
/// transport will cache some connections (up to 5).
- GLOBAL_CONNECTION_LIMIT
+ GP_CONNECTION_LIMIT, ///< Takes long giving number of connections
+ GP_CA_PATH, ///< System path/directory where SSL certs are stored.
+ GP_CA_FILE, ///< System path/file containing certs.
+ GP_HTTP_PROXY ///< String giving host/port to use for HTTP proxy
};
/// Set a parameter on a global policy option. Calls
@@ -122,6 +125,7 @@ class HttpRequest
/// @param value Desired value of option.
/// @return Standard status code.
HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value);
+ HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value);
/// Create a new policy class into which requests can be made.
///
@@ -134,15 +138,15 @@ class HttpRequest
enum EClassPolicy
{
/// Limits the number of connections used for the class.
- CLASS_CONNECTION_LIMIT,
+ CP_CONNECTION_LIMIT,
/// Limits the number of connections used for a single
/// literal address/port pair within the class.
- PER_HOST_CONNECTION_LIMIT,
+ CP_PER_HOST_CONNECTION_LIMIT,
/// Suitable requests are allowed to pipeline on their
/// connections when they ask for it.
- ENABLE_PIPELINING
+ CP_ENABLE_PIPELINING
};
/// Set a parameter on a class-based policy option. Calls
@@ -153,9 +157,7 @@ class HttpRequest
/// @param opt Enum of option to be set.
/// @param value Desired value of option.
/// @return Standard status code.
- HttpStatus setPolicyClassOption(policy_t policy_id,
- EClassPolicy opt,
- long value);
+ HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
/// @}
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index 8e6deb9ccee..7a44415fba9 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -5356,6 +5356,17 @@ void CoreHttp::init()
<< LL_ENDL;
}
+ mRequest = new LLCore::HttpRequest;
+
+ status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE,
+ gDirUtilp->getCAFile());
+ if (! status)
+ {
+ LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: "
+ << status.toString()
+ << LL_ENDL;
+ }
+
status = LLCore::HttpRequest::startThread();
if (! status)
{
@@ -5364,7 +5375,6 @@ void CoreHttp::init()
<< LL_ENDL;
}
- mRequest = new LLCore::HttpRequest;
}
diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp
index 34fb21798fb..f9294b4cd1f 100644
--- a/indra/newview/lltexturefetch.cpp
+++ b/indra/newview/lltexturefetch.cpp
@@ -816,14 +816,6 @@ bool LLTextureFetchWorker::doWork(S32 param)
mFetchTimer.reset();
}
- static LLUUID last_id;
- if (mID != last_id)
- {
- // LL_WARNS("Texture") << "DOWORK SWITCH: " << last_id << " to: " << mID
- // << LL_ENDL;
- last_id = mID;
- }
-
if (mState == INIT)
{
mRawImage = NULL ;
@@ -1109,10 +1101,6 @@ bool LLTextureFetchWorker::doWork(S32 param)
<< " Bytes: " << mRequestedSize
<< " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth
<< LL_ENDL;
-// LL_WARNS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset
-// << " Bytes: " << mRequestedSize
-// << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth
-// << LL_ENDL;
// Will call callbackHttpGet when curl request completes
// *FIXME: enable redirection follow
@@ -1241,7 +1229,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
}
}
- if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded.
+ if (mHaveAllData /* && mRequestedDiscard == 0*/) //the image file is fully loaded.
{
mFileSize = total_size;
}
@@ -1692,13 +1680,32 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response,
body->addRef();
mHttpBufferArray = body;
- if (data_size < mRequestedSize && mRequestedDiscard == 0)
+ if (! partial)
+ {
+ // Response indicates this is the entire asset regardless
+ // of our asking for a byte range. Mark it so and drop
+ // any partial data we might have so that the current
+ // response body becomes the entire dataset.
+ if (data_size <= mRequestedOffset)
+ {
+ LL_WARNS("Texture") << "Fetched entire texture " << mID
+ << " when it was expected to be marked complete. mImageSize: "
+ << mFileSize << " datasize: " << mFormattedImage->getDataSize()
+ << LL_ENDL;
+ }
+ mHaveAllData = TRUE;
+ llassert_always(mDecodeHandle == 0);
+ mFormattedImage = NULL; // discard any previous data we had
+ }
+ else if (data_size < mRequestedSize && mRequestedDiscard == 0)
{
+ // *FIXME: I think we can treat this as complete regardless
+ // of requested discard level. Revisit this...
mHaveAllData = TRUE;
}
else if (data_size > mRequestedSize)
{
- // *TODO: This shouldn't be happening any more
+ // *TODO: This shouldn't be happening any more (REALLY don't expect this anymore)
llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl;
mHaveAllData = TRUE;
llassert_always(mDecodeHandle == 0);
--
GitLab