From 9b410e89c3257d5ab648afef221864ac0a0b4fe7 Mon Sep 17 00:00:00 2001
From: Tofu Linden <tofu.linden@lindenlab.com>
Date: Mon, 1 Feb 2010 12:02:11 +0000
Subject: [PATCH] EXT-4388 Crash in octree line segment intersection code
Yet more sanitization - whew.
---
indra/llrender/llimagegl.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp
index 8bcc4723ae2..36ac3ff119a 100644
--- a/indra/llrender/llimagegl.cpp
+++ b/indra/llrender/llimagegl.cpp
@@ -1693,11 +1693,11 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in)
return;
}
- U32 pick_width = width/2;
- U32 pick_height = height/2;
+ U32 pick_width = width/2 + 1;
+ U32 pick_height = height/2 + 1;
- U32 size = llmax(pick_width, (U32) 1) * llmax(pick_height, (U32) 1);
- size = size/8 + 1;
+ U32 size = pick_width * pick_height;
+ size = (size + 7) / 8; // pixelcount-to-bits
mPickMask = new U8[size];
mPickMaskWidth = pick_width;
mPickMaskHeight = pick_height;
@@ -1745,8 +1745,8 @@ BOOL LLImageGL::getMask(const LLVector2 &tc)
llassert(mPickMaskWidth > 0 && mPickMaskHeight > 0);
- S32 x = (S32)(u * mPickMaskWidth);
- S32 y = (S32)(v * mPickMaskHeight);
+ S32 x = llfloor(u * mPickMaskWidth);
+ S32 y = llfloor(v * mPickMaskHeight);
if (LL_UNLIKELY(x >= mPickMaskWidth))
{
--
GitLab