Skip to content
Snippets Groups Projects
Commit b00813ae authored by Tofu Linden's avatar Tofu Linden
Browse files

EXT-4388 Crash in octree line segment intersection code

a more complete fix and some sanity to prevent recurrance of a similar problem.

reviewed by bao!
parent dbcb6feb
Branches
Tags
No related merge requests found
...@@ -436,6 +436,8 @@ void LLImageGL::init(BOOL usemipmaps) ...@@ -436,6 +436,8 @@ void LLImageGL::init(BOOL usemipmaps)
mLastBindTime = 0.f; mLastBindTime = 0.f;
mPickMask = NULL; mPickMask = NULL;
mPickMaskWidth = 0;
mPickMaskHeight = 0;
mUseMipMaps = usemipmaps; mUseMipMaps = usemipmaps;
mHasExplicitFormat = FALSE; mHasExplicitFormat = FALSE;
mAutoGenMips = FALSE; mAutoGenMips = FALSE;
...@@ -528,6 +530,11 @@ void LLImageGL::setSize(S32 width, S32 height, S32 ncomponents) ...@@ -528,6 +530,11 @@ void LLImageGL::setSize(S32 width, S32 height, S32 ncomponents)
destroyGLTexture(); destroyGLTexture();
} }
// pickmask validity depends on old image size, delete it
delete [] mPickMask;
mPickMask = NULL;
mPickMaskWidth = mPickMaskHeight = 0;
mWidth = width; mWidth = width;
mHeight = height; mHeight = height;
mComponents = ncomponents; mComponents = ncomponents;
...@@ -1675,12 +1682,14 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in) ...@@ -1675,12 +1682,14 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in)
return ; return ;
} }
delete [] mPickMask;
mPickMask = NULL;
mPickMaskWidth = mPickMaskHeight = 0;
if (mFormatType != GL_UNSIGNED_BYTE || if (mFormatType != GL_UNSIGNED_BYTE ||
mFormatPrimary != GL_RGBA) mFormatPrimary != GL_RGBA)
{ {
//cannot generate a pick mask for this texture //cannot generate a pick mask for this texture
delete [] mPickMask;
mPickMask = NULL;
return; return;
} }
...@@ -1688,11 +1697,10 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in) ...@@ -1688,11 +1697,10 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in)
U32 pick_height = height/2; U32 pick_height = height/2;
U32 size = llmax(pick_width, (U32) 1) * llmax(pick_height, (U32) 1); U32 size = llmax(pick_width, (U32) 1) * llmax(pick_height, (U32) 1);
size = size/8 + 1; size = size/8 + 1;
delete[] mPickMask;
mPickMask = new U8[size]; mPickMask = new U8[size];
mPickMaskWidth = pick_width;
mPickMaskHeight = pick_height;
memset(mPickMask, 0, sizeof(U8) * size); memset(mPickMask, 0, sizeof(U8) * size);
...@@ -1727,35 +1735,34 @@ BOOL LLImageGL::getMask(const LLVector2 &tc) ...@@ -1727,35 +1735,34 @@ BOOL LLImageGL::getMask(const LLVector2 &tc)
if (mPickMask) if (mPickMask)
{ {
S32 width = getWidth()/2;
S32 height = getHeight()/2;
F32 u = tc.mV[0] - floorf(tc.mV[0]); F32 u = tc.mV[0] - floorf(tc.mV[0]);
F32 v = tc.mV[1] - floorf(tc.mV[1]); F32 v = tc.mV[1] - floorf(tc.mV[1]);
if (u < 0.f || u > 1.f || if (LL_UNLIKELY(u < 0.f || u > 1.f ||
v < 0.f || v > 1.f) v < 0.f || v > 1.f))
{ {
LL_WARNS_ONCE("render") << "Ugh, u/v out of range in image mask pick" << LL_ENDL; LL_WARNS_ONCE("render") << "Ugh, u/v out of range in image mask pick" << LL_ENDL;
u = v = 0.f; u = v = 0.f;
llassert(false); llassert(false);
} }
S32 x = (S32)(u * width); llassert(mPickMaskWidth > 0 && mPickMaskHeight > 0);
S32 y = (S32)(v * height);
S32 x = (S32)(u * mPickMaskWidth);
S32 y = (S32)(v * mPickMaskHeight);
if (x >= width) if (LL_UNLIKELY(x >= mPickMaskWidth))
{ {
LL_WARNS_ONCE("render") << "Ooh, width overrun on pick mask read, that coulda been bad." << LL_ENDL; LL_WARNS_ONCE("render") << "Ooh, width overrun on pick mask read, that coulda been bad." << LL_ENDL;
x = llmax(0, width-1); x = llmax(0, mPickMaskWidth-1);
} }
if (y >= height) if (LL_UNLIKELY(y >= mPickMaskHeight))
{ {
LL_WARNS_ONCE("render") << "Ooh, height overrun on pick mask read, that woulda been bad." << LL_ENDL; LL_WARNS_ONCE("render") << "Ooh, height overrun on pick mask read, that woulda been bad." << LL_ENDL;
y = llmax(0, height-1); y = llmax(0, mPickMaskHeight-1);
} }
S32 idx = y*width+x; S32 idx = y*mPickMaskWidth+x;
S32 offset = idx%8; S32 offset = idx%8;
res = mPickMask[idx/8] & (1 << offset) ? TRUE : FALSE; res = mPickMask[idx/8] & (1 << offset) ? TRUE : FALSE;
......
...@@ -193,6 +193,8 @@ class LLImageGL : public LLRefCount ...@@ -193,6 +193,8 @@ class LLImageGL : public LLRefCount
private: private:
LLPointer<LLImageRaw> mSaveData; // used for destroyGL/restoreGL LLPointer<LLImageRaw> mSaveData; // used for destroyGL/restoreGL
U8* mPickMask; //downsampled bitmap approximation of alpha channel. NULL if no alpha channel U8* mPickMask; //downsampled bitmap approximation of alpha channel. NULL if no alpha channel
U16 mPickMaskWidth;
U16 mPickMaskHeight;
S8 mUseMipMaps; S8 mUseMipMaps;
S8 mHasExplicitFormat; // If false (default), GL format is f(mComponents) S8 mHasExplicitFormat; // If false (default), GL format is f(mComponents)
S8 mAutoGenMips; S8 mAutoGenMips;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment