llassetstorage.cpp

/** 
 * @file llassetstorage.cpp
 * @brief Implementation of the base asset storage system.
 *
 * $LicenseInfo:firstyear=2001&license=viewerlgpl$
 * Second Life Viewer Source Code
 * Copyright (C) 2010, Linden Research, Inc.
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation;
 * version 2.1 of the License only.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 * 
 * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
 * $/LicenseInfo$
 */

#include "linden_common.h"

// system library includes
#include <sys/types.h>
#include <sys/stat.h>

#include "llassetstorage.h"

// linden library includes
#include "llmath.h"
#include "llstring.h"
#include "lldir.h"
#include "llsd.h"
#include "llframetimer.h"

// this library includes
#include "message.h"
#include "llxfermanager.h"
#include "llvfile.h"
#include "llvfs.h"
#include "lldbstrings.h"

#include "lltransfersourceasset.h"
#include "lltransfertargetvfile.h" // For debugging

#include "llmetrics.h"
#include "lltrace.h"

LLAssetStorage *gAssetStorage = NULL;
LLMetrics *LLAssetStorage::metric_recipient = NULL;

static LLTrace::CountStatHandle<> sFailedDownloadCount("faileddownloads", "Number of times LLAssetStorage::getAssetData() has failed");


const LLUUID CATEGORIZE_LOST_AND_FOUND_ID(std::string("00000000-0000-0000-0000-000000000010"));

const U64 TOXIC_ASSET_LIFETIME = (120 * 1000000);       // microseconds

namespace
{
    bool operator == (const LLAssetStorage::LLGetAssetCallback &lhs, const LLAssetStorage::LLGetAssetCallback &rhs)
    {
        auto fnPtrLhs = lhs.target<LLAssetStorage::LLGetAssetCallback>();
        auto fnPtrRhs = rhs.target<LLAssetStorage::LLGetAssetCallback>();
        if (fnPtrLhs && fnPtrRhs)
            return (*fnPtrLhs == *fnPtrRhs);
        else if (!fnPtrLhs && !fnPtrRhs)
            return true;
        return false;
    }

// Rider: This is the general case of the operator declared above. The code compares the callback 
// passed into the LLAssetStorage functions to determine if there are duplicated requests for an 
// asset.  Unfortunately std::function does not provide a direct way to compare two variables so 
// we define the operator here. 
// XCode is not very happy with the variadic temples in use below so we will just define the specific 
// case of comparing two LLGetAssetCallback objects since that is all we really use.
// 
//     template<typename T, typename... U>
//     bool operator == (const std::function<T(U...)> &a, const std::function <T(U...)> &b)
//     {
//         typedef T(fnType)(U...);
// 
//         auto fnPtrA = a.target<T(*)(U...)>();
//         auto fnPtrB = b.target<T(*)(U...)>();
//         if (fnPtrA && fnPtrB)
//             return (*fnPtrA == *fnPtrB);
//         else if (!fnPtrA && !fnPtrB)
//             return true;
//         return false;
//     }

}

///----------------------------------------------------------------------------
/// LLAssetInfo
///----------------------------------------------------------------------------

LLAssetInfo::LLAssetInfo( void )
    :   mDescription(),
        mName(),
        mUuid(),
        mCreatorID(),
        mType( LLAssetType::AT_NONE )
{ }

LLAssetInfo::LLAssetInfo( const LLUUID& object_id, const LLUUID& creator_id,
                          LLAssetType::EType type, const char* name,
                          const char* desc )
    :   mUuid( object_id ), 
        mCreatorID( creator_id ), 
        mType( type )
{
    setName( name );
    setDescription( desc );
}

LLAssetInfo::LLAssetInfo( const LLNameValue& nv )
{
    setFromNameValue( nv );
}

// make sure the name is short enough, and strip all pipes since they
// are reserved characters in our inventory tracking system.
void LLAssetInfo::setName( const std::string& name )
{
    if( !name.empty() )
    {
        mName.assign( name, 0, llmin((U32)name.size(), (U32)DB_INV_ITEM_NAME_STR_LEN) );
        mName.erase( std::remove(mName.begin(), mName.end(), '|'),
                     mName.end() );
    }
}

// make sure the name is short enough, and strip all pipes since they
// are reserved characters in our inventory tracking system.
void LLAssetInfo::setDescription( const std::string& desc )
{
    if( !desc.empty() )
    {
        mDescription.assign( desc, 0, llmin((U32)desc.size(),
                                            (U32)DB_INV_ITEM_DESC_STR_LEN) );
        mDescription.erase( std::remove(mDescription.begin(),
                                        mDescription.end(), '|'),
                            mDescription.end() );
    }
}

// Assets (aka potential inventory items) can be applied to an
// object in the world. We'll store that as a string name value
// pair where the name encodes part of asset info, and the value
// the rest.  LLAssetInfo objects will be responsible for parsing
// the meaning out froman LLNameValue object. See the inventory
// design docs for details. Briefly:
//   name=<inv_type>|<uuid>
//   value=<creatorid>|<name>|<description>|
void LLAssetInfo::setFromNameValue( const LLNameValue& nv )
{
    std::string str;
    std::string buf;
    std::string::size_type pos1;
    std::string::size_type pos2;

    // convert the name to useful information
    str.assign( nv.mName );
    pos1 = str.find('|');
    buf.assign( str, 0, pos1++ );
    mType = LLAssetType::lookup( buf );
    buf.assign( str, pos1, std::string::npos );
    mUuid.set( buf );

    // convert the value to useful information
    str.assign( nv.getAsset() );
    pos1 = str.find('|');
    buf.assign( str, 0, pos1++ );
    mCreatorID.set( buf );
    pos2 = str.find( '|', pos1 );
    buf.assign( str, pos1, (pos2++) - pos1 );
    setName( buf );
    buf.assign( str, pos2, std::string::npos );
    setDescription( buf );
    LL_DEBUGS("AssetStorage") << "uuid: " << mUuid << LL_ENDL;
    LL_DEBUGS("AssetStorage") << "creator: " << mCreatorID << LL_ENDL;
}

///----------------------------------------------------------------------------
/// LLBaseDownloadRequest
///----------------------------------------------------------------------------

LLBaseDownloadRequest::LLBaseDownloadRequest(const LLUUID &uuid, const LLAssetType::EType type)
    : mUUID(uuid),
      mType(type),
      mDownCallback(),
      mUserData(NULL),
      mHost(),
      mIsTemp(FALSE),
      mIsPriority(FALSE),
      mDataSentInFirstPacket(FALSE),
      mDataIsInVFS(FALSE)
{
    // Need to guarantee that this time is up to date, we may be creating a circuit even though we haven't been
    //  running a message system loop.
    mTime = LLMessageSystem::getMessageTimeSeconds(TRUE);
}

// virtual
LLBaseDownloadRequest* LLBaseDownloadRequest::getCopy()
{
    return new LLBaseDownloadRequest(*this);
}


///----------------------------------------------------------------------------
/// LLAssetRequest
///----------------------------------------------------------------------------

LLAssetRequest::LLAssetRequest(const LLUUID &uuid, const LLAssetType::EType type)
    :   LLBaseDownloadRequest(uuid, type),
        mUpCallback(),
        mInfoCallback( NULL ),
        mIsLocal(FALSE),
        mIsUserWaiting(FALSE),
        mTimeout(LL_ASSET_STORAGE_TIMEOUT),
        mBytesFetched(0)
{
}

// virtual
LLSD LLAssetRequest::getTerseDetails() const
{
    LLSD sd;
    sd["asset_id"] = getUUID();
    sd["type_long"] = LLAssetType::lookupHumanReadable(getType());
    sd["type"] = LLAssetType::lookup(getType());
    sd["time"] = mTime.value();
    time_t timestamp = (time_t) mTime.value();
    std::ostringstream time_string;
    time_string << ctime(&timestamp);
    sd["time_string"] = time_string.str();
    return sd;
}

// virtual
LLSD LLAssetRequest::getFullDetails() const
{
    LLSD sd = getTerseDetails();
    sd["host"] = mHost.getIPandPort();
    sd["requesting_agent"] = mRequestingAgentID;
    sd["is_temp"] = mIsTemp;
    sd["is_local"] = mIsLocal;
    sd["is_priority"] = mIsPriority;
    sd["data_send_in_first_packet"] = mDataSentInFirstPacket;
    sd["data_is_in_vfs"] = mDataIsInVFS;

    return sd;
}

LLBaseDownloadRequest* LLAssetRequest::getCopy()
{
    return new LLAssetRequest(*this);
}

///----------------------------------------------------------------------------
/// LLInvItemRequest
///----------------------------------------------------------------------------

LLInvItemRequest::LLInvItemRequest(const LLUUID &uuid, const LLAssetType::EType type)
    :   LLBaseDownloadRequest(uuid, type)
{
}

LLBaseDownloadRequest* LLInvItemRequest::getCopy()
{
    return new LLInvItemRequest(*this);
}

///----------------------------------------------------------------------------
/// LLEstateAssetRequest
///----------------------------------------------------------------------------

LLEstateAssetRequest::LLEstateAssetRequest(const LLUUID &uuid, const LLAssetType::EType atype,
                                           EstateAssetType etype)
    :   LLBaseDownloadRequest(uuid, atype),
        mEstateAssetType(etype)
{
}

LLBaseDownloadRequest* LLEstateAssetRequest::getCopy()
{
    return new LLEstateAssetRequest(*this);
}


///----------------------------------------------------------------------------
/// LLAssetStorage
///----------------------------------------------------------------------------

// since many of these functions are called by the messaging and xfer systems,
// they are declared as static and are passed a "this" handle
// it's a C/C++ mish-mash!

// TODO: permissions on modifications - maybe don't allow at all?
// TODO: verify that failures get propogated down
// TODO: rework tempfile handling?


LLAssetStorage::LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, LLVFS *vfs, LLVFS *static_vfs, const LLHost &upstream_host)
{
    _init(msg, xfer, vfs, static_vfs, upstream_host);
}

LLAssetStorage::LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer,
                               LLVFS *vfs, LLVFS *static_vfs)
{
    _init(msg, xfer, vfs, static_vfs, LLHost());
}

void LLAssetStorage::_init(LLMessageSystem *msg,
                           LLXferManager *xfer,
                           LLVFS *vfs,
                           LLVFS *static_vfs,
                           const LLHost &upstream_host)
{
    mShutDown = FALSE;
    mMessageSys = msg;
    mXferManager = xfer;
    mVFS = vfs;
    mStaticVFS = static_vfs;

    setUpstream(upstream_host);
    msg->setHandlerFuncFast(_PREHASH_AssetUploadComplete, processUploadComplete, (void **)this);
}

LLAssetStorage::~LLAssetStorage()
{
    mShutDown = TRUE;
    
    _cleanupRequests(TRUE, LL_ERR_CIRCUIT_GONE);

    if (gMessageSystem)
    {
        // Warning!  This won't work if there's more than one asset storage.
        // unregister our callbacks with the message system
        gMessageSystem->setHandlerFuncFast(_PREHASH_AssetUploadComplete, NULL, NULL);
    }

    // Clear the toxic asset map
    mToxicAssetMap.clear();
}

void LLAssetStorage::setUpstream(const LLHost &upstream_host)
{
    LL_DEBUGS("AppInit") << "AssetStorage: Setting upstream provider to " << upstream_host << LL_ENDL;
    
    mUpstreamHost = upstream_host;
}

void LLAssetStorage::checkForTimeouts()
{
    _cleanupRequests(FALSE, LL_ERR_TCP_TIMEOUT);
}

void LLAssetStorage::_cleanupRequests(BOOL all, S32 error)
{
    F64Seconds mt_secs = LLMessageSystem::getMessageTimeSeconds();

    request_list_t timed_out;
    S32 rt;
    for (rt = 0; rt < RT_COUNT; rt++)
    {
        request_list_t* requests = getRequestList((ERequestType)rt);
        for (request_list_t::iterator iter = requests->begin();
             iter != requests->end(); )
        {
            request_list_t::iterator curiter = iter++;
            LLAssetRequest* tmp = *curiter;
            // if all is true, we want to clean up everything
            // otherwise just check for timed out requests
            // EXCEPT for upload timeouts
            if (all 
                || ((RT_DOWNLOAD == rt)
                    && LL_ASSET_STORAGE_TIMEOUT < (mt_secs - tmp->mTime)))
            {
                LL_WARNS("AssetStorage") << "Asset " << getRequestName((ERequestType)rt) << " request "
                                         << (all ? "aborted" : "timed out") << " for "
                                         << tmp->getUUID() << "."
                                         << LLAssetType::lookup(tmp->getType()) << LL_ENDL;
                
                timed_out.push_front(tmp);
                iter = requests->erase(curiter);
            }
        }
    }

    LLAssetInfo     info;
    for (request_list_t::iterator iter = timed_out.begin();
         iter != timed_out.end();  )
    {
        request_list_t::iterator curiter = iter++;
        LLAssetRequest* tmp = *curiter;
        if (tmp->mUpCallback)
        {
            tmp->mUpCallback(tmp->getUUID(), tmp->mUserData, error, LLExtStat::NONE);
        }
        if (tmp->mDownCallback)
        {
            tmp->mDownCallback(mVFS, tmp->getUUID(), tmp->getType(), tmp->mUserData, error, LLExtStat::NONE);
        }
        if (tmp->mInfoCallback)
        {
            tmp->mInfoCallback(&info, tmp->mUserData, error);
        }
        delete tmp;
    }

}

BOOL LLAssetStorage::hasLocalAsset(const LLUUID &uuid, const LLAssetType::EType type)
{
    return mStaticVFS->getExists(uuid, type) || mVFS->getExists(uuid, type);
}

bool LLAssetStorage::findInStaticVFSAndInvokeCallback(const LLUUID& uuid, LLAssetType::EType type,
                                                      LLGetAssetCallback callback, void *user_data)
{
    if (user_data)
    {
        // The *user_data should not be passed without a callback to clean it up.
        llassert(callback != NULL);
    }

    BOOL exists = mStaticVFS->getExists(uuid, type);
    if (exists)
    {
        LLVFile file(mStaticVFS, uuid, type);
        U32 size = file.getSize();
        if (size > 0)
        {
            // we've already got the file
            if (callback)
            {
                callback(mStaticVFS, uuid, type, user_data, LL_ERR_NOERR, LLExtStat::VFS_CACHED);
            }
            return true;
        }
        else
        {
            LL_WARNS("AssetStorage") << "Asset vfile " << uuid << ":" << type
                                     << " found in static cache with bad size " << file.getSize() << ", ignoring" << LL_ENDL;
        }
    }
    return false;
}

///////////////////////////////////////////////////////////////////////////
// GET routines
///////////////////////////////////////////////////////////////////////////

// IW - uuid is passed by value to avoid side effects, please don't re-add &    
void LLAssetStorage::getAssetData(const LLUUID uuid,
                                  LLAssetType::EType type, 
                                  LLAssetStorage::LLGetAssetCallback callback,
                                  void *user_data, 
                                  BOOL is_priority)
{
    LL_DEBUGS("AssetStorage") << "LLAssetStorage::getAssetData() - " << uuid << "," << LLAssetType::lookup(type) << LL_ENDL;

    LL_DEBUGS("AssetStorage") << "ASSET_TRACE requesting " << uuid << " type " << LLAssetType::lookup(type) << LL_ENDL;

    if (user_data)
    {
        // The *user_data should not be passed without a callback to clean it up.
        llassert(callback != NULL);
    }

    if (mShutDown)
    {
        LL_DEBUGS("AssetStorage") << "ASSET_TRACE cancelled " << uuid << " type " << LLAssetType::lookup(type) << " shutting down" << LL_ENDL;

        if (callback)
        {
            add(sFailedDownloadCount, 1);
            callback(mVFS, uuid, type, user_data, LL_ERR_ASSET_REQUEST_FAILED, LLExtStat::NONE);
        }
        return;
    }

    if (uuid.isNull())
    {
        // Special case early out for NULL uuid and for shutting down
        if (callback)
        {
            add(sFailedDownloadCount, 1);
            callback(mVFS, uuid, type, user_data, LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE, LLExtStat::NULL_UUID);
        }
        return;
    }

    // Try static VFS first.
    if (findInStaticVFSAndInvokeCallback(uuid,type,callback,user_data))
    {
        LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << uuid << " found in static VFS" << LL_ENDL;
        return;
    }

    BOOL exists = mVFS->getExists(uuid, type);
    LLVFile file(mVFS, uuid, type);
    U32 size = exists ? file.getSize() : 0;

    if (size > 0)
    {
        // we've already got the file
        // theoretically, partial files w/o a pending request shouldn't happen
        // unless there's a weird error
        if (callback)
        {
            callback(mVFS, uuid, type, user_data, LL_ERR_NOERR, LLExtStat::VFS_CACHED);
        }

        LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << uuid << " found in VFS" << LL_ENDL;
    }
    else
    {
        if (exists)
        {
            LL_WARNS("AssetStorage") << "Asset vfile " << uuid << ":" << type << " found with bad size " << file.getSize() << ", removing" << LL_ENDL;
            file.remove();
        }
        
        BOOL duplicate = FALSE;
        
        // check to see if there's a pending download of this uuid already
        for (request_list_t::iterator iter = mPendingDownloads.begin();
             iter != mPendingDownloads.end(); ++iter )
        {
            LLAssetRequest  *tmp = *iter;
            if ((type == tmp->getType()) && (uuid == tmp->getUUID()))
            {
                if (callback == tmp->mDownCallback && user_data == tmp->mUserData)
                {
                    // this is a duplicate from the same subsystem - throw it away
                    LL_WARNS("AssetStorage") << "Discarding duplicate request for asset " << uuid
                                             << "." << LLAssetType::lookup(type) << LL_ENDL;
                    return;
                }
                
                // this is a duplicate request
                // queue the request, but don't actually ask for it again
                duplicate = TRUE;
            }
        }
        if (duplicate)
        {
            LL_DEBUGS("AssetStorage") << "Adding additional non-duplicate request for asset " << uuid 
                                      << "." << LLAssetType::lookup(type) << LL_ENDL;
        }
        
        _queueDataRequest(uuid, type, callback, user_data, duplicate, is_priority);     
    }
}

// static
void LLAssetStorage::removeAndCallbackPendingDownloads(const LLUUID& file_id, LLAssetType::EType file_type,
                                                       const LLUUID& callback_id, LLAssetType::EType callback_type,
                                                       S32 result_code, LLExtStat ext_status)
{
    // find and callback ALL pending requests for this UUID
    // SJB: We process the callbacks in reverse order, I do not know if this is important,
    //      but I didn't want to mess with it.
    request_list_t requests;
    for (request_list_t::iterator iter = gAssetStorage->mPendingDownloads.begin();
         iter != gAssetStorage->mPendingDownloads.end();  )
    {
        request_list_t::iterator curiter = iter++;
        LLAssetRequest* tmp = *curiter;
        if ((tmp->getUUID() == file_id) && (tmp->getType()== file_type))
        {
            requests.push_front(tmp);
            iter = gAssetStorage->mPendingDownloads.erase(curiter);
        }
    }
    for (request_list_t::iterator iter = requests.begin();
         iter != requests.end();  )
    {
        request_list_t::iterator curiter = iter++;
        LLAssetRequest* tmp = *curiter;
        if (tmp->mDownCallback)
        {
            if (result_code!= LL_ERR_NOERR)
            {
                add(sFailedDownloadCount, 1);
            }
            tmp->mDownCallback(gAssetStorage->mVFS, callback_id, callback_type, tmp->mUserData, result_code, ext_status);
        }
        delete tmp;
    }
}

void LLAssetStorage::downloadCompleteCallback(
    S32 result,
    const LLUUID& file_id,
    LLAssetType::EType file_type,
    LLBaseDownloadRequest* user_data,
    LLExtStat ext_status)
{
    LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << file_id << " downloadCompleteCallback" << LL_ENDL;

    LL_DEBUGS("AssetStorage") << "LLAssetStorage::downloadCompleteCallback() for " << file_id
                              << "," << LLAssetType::lookup(file_type) << LL_ENDL;
    LLAssetRequest* req = (LLAssetRequest*)user_data;
    if(!req)
    {
        LL_WARNS("AssetStorage") << "LLAssetStorage::downloadCompleteCallback called without"
            "a valid request." << LL_ENDL;
        return;
    }
    if (!gAssetStorage)
    {
        LL_WARNS("AssetStorage") << "LLAssetStorage::downloadCompleteCallback called without any asset system, aborting!" << LL_ENDL;
        return;
    }

    LLUUID callback_id;
    LLAssetType::EType callback_type;

    // Inefficient since we're doing a find through a list that may have thousands of elements.
    // This is due for refactoring; we will probably change mPendingDownloads into a set.
    request_list_t::iterator download_iter = std::find(gAssetStorage->mPendingDownloads.begin(), 
                                                       gAssetStorage->mPendingDownloads.end(),
                                                       req);

    if (download_iter != gAssetStorage->mPendingDownloads.end())
    {
        callback_id = file_id;
        callback_type = file_type;
    }
    else
    {
        // either has already been deleted by _cleanupRequests or it's a transfer.
        callback_id = req->getUUID();
        callback_type = req->getType();
    }

    if (LL_ERR_NOERR == result)
    {
        // we might have gotten a zero-size file
        LLVFile vfile(gAssetStorage->mVFS, callback_id, callback_type);
        if (vfile.getSize() <= 0)
        {
            LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset " << callback_id << LL_ENDL;
            
            result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE;
            vfile.remove();
        }
        else
        {
#if 1
            for (request_list_t::iterator iter = gAssetStorage->mPendingDownloads.begin();
                 iter != gAssetStorage->mPendingDownloads.end(); ++iter  )
            {
                LLAssetRequest* dlreq = *iter;
                if ((dlreq->getUUID() == file_id) && (dlreq->getType()== file_type))
                {
                    dlreq->mBytesFetched = vfile.getSize();
                }
            }
#endif
        }
    }

    removeAndCallbackPendingDownloads(file_id, file_type, callback_id, callback_type, result, ext_status);
}

void LLAssetStorage::getEstateAsset(
    const LLHost &object_sim,
    const LLUUID &agent_id, 
    const LLUUID &session_id,
    const LLUUID &asset_id, 
    LLAssetType::EType atype, 
    EstateAssetType etype,
    LLGetAssetCallback callback, 
    void *user_data, 
    BOOL is_priority)
{
    LL_DEBUGS() << "LLAssetStorage::getEstateAsset() - " << asset_id << "," << LLAssetType::lookup(atype) << ", estatetype " << etype << LL_ENDL;

    //
    // Probably will get rid of this early out?
    //
    if (asset_id.isNull())
    {
        // Special case early out for NULL uuid
        if (callback)
        {
            add(sFailedDownloadCount, 1);
            callback(mVFS, asset_id, atype, user_data, LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE, LLExtStat::NULL_UUID);
        }
        return;
    }

    // Try static VFS first.
    if (findInStaticVFSAndInvokeCallback(asset_id,atype,callback,user_data))
    {
        return;
    }
    
    BOOL exists = mVFS->getExists(asset_id, atype);
    LLVFile file(mVFS, asset_id, atype);
    U32 size = exists ? file.getSize() : 0;

    if (size > 0)
    {
        // we've already got the file
        // theoretically, partial files w/o a pending request shouldn't happen
        // unless there's a weird error
        if (callback)
        {
            callback(mVFS, asset_id, atype, user_data, LL_ERR_NOERR, LLExtStat::VFS_CACHED);
        }
    }
    else
    {
        if (exists)
        {
            LL_WARNS("AssetStorage") << "Asset vfile " << asset_id << ":" << atype << " found with bad size " << file.getSize() << ", removing" << LL_ENDL;
            file.remove();
        }

        // See whether we should talk to the object's originating sim, or the upstream provider.
        LLHost source_host;
        if (object_sim.isOk())
        {
            source_host = object_sim;
        }
        else
        {
            source_host = mUpstreamHost;
        }
        if (source_host.isOk())
        {
            // stash the callback info so we can find it after we get the response message
            LLEstateAssetRequest req(asset_id, atype, etype);
            req.mDownCallback = callback;
            req.mUserData = user_data;
            req.mIsPriority = is_priority;

            // send request message to our upstream data provider
            // Create a new asset transfer.
            LLTransferSourceParamsEstate spe;
            spe.setAgentSession(agent_id, session_id);
            spe.setEstateAssetType(etype);

            // Set our destination file, and the completion callback.
            LLTransferTargetParamsVFile tpvf;
            tpvf.setAsset(asset_id, atype);
            tpvf.setCallback(downloadEstateAssetCompleteCallback, req);

            LL_DEBUGS("AssetStorage") << "Starting transfer for " << asset_id << LL_ENDL;
            LLTransferTargetChannel *ttcp = gTransferManager.getTargetChannel(source_host, LLTCT_ASSET);
            ttcp->requestTransfer(spe, tpvf, 100.f + (is_priority ? 1.f : 0.f));
        }
        else
        {
            // uh-oh, we shouldn't have gotten here
            LL_WARNS("AssetStorage") << "Attempt to move asset data request upstream w/o valid upstream provider" << LL_ENDL;
            if (callback)
            {
                add(sFailedDownloadCount, 1);
                callback(mVFS, asset_id, atype, user_data, LL_ERR_CIRCUIT_GONE, LLExtStat::NO_UPSTREAM);
            }
        }
    }
}

void LLAssetStorage::downloadEstateAssetCompleteCallback(
    S32 result,
    const LLUUID& file_id,
    LLAssetType::EType file_type,
    LLBaseDownloadRequest* user_data,
    LLExtStat ext_status)
{
    LLEstateAssetRequest *req = (LLEstateAssetRequest*)user_data;
    if(!req)
    {
        LL_WARNS("AssetStorage") << "LLAssetStorage::downloadEstateAssetCompleteCallback called"
            " without a valid request." << LL_ENDL;
        return;
    }
    if (!gAssetStorage)
    {
        LL_WARNS("AssetStorage") << "LLAssetStorage::downloadEstateAssetCompleteCallback called"
            " without any asset system, aborting!" << LL_ENDL;
        return;
    }

    req->setUUID(file_id);
    req->setType(file_type);
    if (LL_ERR_NOERR == result)
    {
        // we might have gotten a zero-size file
        LLVFile vfile(gAssetStorage->mVFS, req->getUUID(), req->getAType());
        if (vfile.getSize() <= 0)
        {
            LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset!" << LL_ENDL;

            result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE;
            vfile.remove();
        }
    }

    if (result != LL_ERR_NOERR)
    {
        add(sFailedDownloadCount, 1);
    }
    req->mDownCallback(gAssetStorage->mVFS, req->getUUID(), req->getAType(), req->mUserData, result, ext_status);
}

void LLAssetStorage::getInvItemAsset(
    const LLHost &object_sim, 
    const LLUUID &agent_id, 
    const LLUUID &session_id,
    const LLUUID &owner_id,
    const LLUUID &task_id, 
    const LLUUID &item_id,
    const LLUUID &asset_id, 
    LLAssetType::EType atype,
    LLGetAssetCallback callback, 
    void *user_data, 
    BOOL is_priority)
{
    LL_DEBUGS() << "LLAssetStorage::getInvItemAsset() - " << asset_id << "," << LLAssetType::lookup(atype) << LL_ENDL;

    bool exists = false; 
    U32 size = 0;

    if(asset_id.notNull())
    {
        // Try static VFS first.
        if (findInStaticVFSAndInvokeCallback( asset_id, atype, callback, user_data))
        {
            return;
        }

        exists = mVFS->getExists(asset_id, atype);
        LLVFile file(mVFS, asset_id, atype);
        size = exists ? file.getSize() : 0;
        if(exists && size < 1)
        {
            LL_WARNS("AssetStorage") << "Asset vfile " << asset_id << ":" << atype << " found with bad size " << file.getSize() << ", removing" << LL_ENDL;
            file.remove();
        }

    }

    if (size > 0)
    {
        // we've already got the file
        // theoretically, partial files w/o a pending request shouldn't happen
        // unless there's a weird error
        if (callback)
        {
            callback(mVFS, asset_id, atype, user_data, LL_ERR_NOERR, LLExtStat::VFS_CACHED);
        }
    }
    else
    {
        // See whether we should talk to the object's originating sim,
        // or the upstream provider.
        LLHost source_host;
        if (object_sim.isOk())
        {
            source_host = object_sim;
        }
        else
        {
            source_host = mUpstreamHost;
        }
        if (source_host.isOk())
        {
            // stash the callback info so we can find it after we get the response message
            LLInvItemRequest req(asset_id, atype);
            req.mDownCallback = callback;
            req.mUserData = user_data;
            req.mIsPriority = is_priority;

            // send request message to our upstream data provider
            // Create a new asset transfer.
            LLTransferSourceParamsInvItem spi;
            spi.setAgentSession(agent_id, session_id);
            spi.setInvItem(owner_id, task_id, item_id);
            spi.setAsset(asset_id, atype);

            LL_DEBUGS("ViewerAsset") << "requesting inv item id " << item_id << " asset_id " << asset_id << " type " << LLAssetType::lookup(atype) << LL_ENDL;
            
            // Set our destination file, and the completion callback.
            LLTransferTargetParamsVFile tpvf;
            tpvf.setAsset(asset_id, atype);
            tpvf.setCallback(downloadInvItemCompleteCallback, req);

            LL_DEBUGS("AssetStorage") << "Starting transfer for inventory asset "
                                      << item_id << " owned by " << owner_id << "," << task_id
                                      << LL_ENDL;
            LLTransferTargetChannel *ttcp = gTransferManager.getTargetChannel(source_host, LLTCT_ASSET);
            ttcp->requestTransfer(spi, tpvf, 100.f + (is_priority ? 1.f : 0.f));
        }
        else
        {
            // uh-oh, we shouldn't have gotten here
            LL_WARNS("AssetStorage") << "Attempt to move asset data request upstream w/o valid upstream provider" << LL_ENDL;
            if (callback)
            {
                add(sFailedDownloadCount, 1);
                callback(mVFS, asset_id, atype, user_data, LL_ERR_CIRCUIT_GONE, LLExtStat::NO_UPSTREAM);
            }
        }
    }
}


void LLAssetStorage::downloadInvItemCompleteCallback(
    S32 result,
    const LLUUID& file_id,
    LLAssetType::EType file_type,
    LLBaseDownloadRequest* user_data,
    LLExtStat ext_status)
{
    LLInvItemRequest *req = (LLInvItemRequest*)user_data;
    if(!req)
    {
        LL_WARNS("AssetStorage") << "LLAssetStorage::downloadEstateAssetCompleteCallback called"
            " without a valid request." << LL_ENDL;
        return;
    }
    if (!gAssetStorage)
    {
        LL_WARNS("AssetStorage") << "LLAssetStorage::downloadCompleteCallback called without any asset system, aborting!" << LL_ENDL;
        return;
    }

    req->setUUID(file_id);
    req->setType(file_type);
    if (LL_ERR_NOERR == result)
    {
        // we might have gotten a zero-size file
        LLVFile vfile(gAssetStorage->mVFS, req->getUUID(), req->getType());
        if (vfile.getSize() <= 0)
        {
            LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset!" << LL_ENDL;

            result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE;
            vfile.remove();
        }
    }

    if (result != LL_ERR_NOERR)
    {
        add(sFailedDownloadCount, 1);
    }
    req->mDownCallback(gAssetStorage->mVFS, req->getUUID(), req->getType(), req->mUserData, result, ext_status);
}

/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Store routines
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////

// static
void LLAssetStorage::uploadCompleteCallback(
    const LLUUID& uuid, 
    void *user_data, 
    S32 result, 
    LLExtStat ext_status) // StoreAssetData callback (fixed)
{
    if (!gAssetStorage)
    {
        LL_WARNS("AssetStorage") << "LLAssetStorage::uploadCompleteCallback has no gAssetStorage!" << LL_ENDL;
        return;
    }
    LLAssetRequest  *req     = (LLAssetRequest *)user_data;
    BOOL            success  = TRUE;

    if (result)
    {
        LL_WARNS("AssetStorage") << "LLAssetStorage::uploadCompleteCallback " << result << ":" << getErrorString(result) << " trying to upload file to upstream provider" << LL_ENDL;
        success = FALSE;
    }

    // we're done grabbing the file, tell the client
    gAssetStorage->mMessageSys->newMessageFast(_PREHASH_AssetUploadComplete);
    gAssetStorage->mMessageSys->nextBlockFast(_PREHASH_AssetBlock);
    gAssetStorage->mMessageSys->addUUIDFast(_PREHASH_UUID, uuid);
    gAssetStorage->mMessageSys->addS8Fast(_PREHASH_Type, req->getType());
    gAssetStorage->mMessageSys->addBOOLFast(_PREHASH_Success, success);
    gAssetStorage->mMessageSys->sendReliable(req->mHost);