libstagefright: fix overflow in pvdec_api.cpp.

Bug: 20674086
Change-Id: Ie2c711865c3b92f3fa2f3c7a436fa0e3687eb8b3
(cherry picked from commit d7bb1cd786e5ea4ac61119cc1a08082474f7787b)

diff --git a/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp b/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
index 419e3f4..30d5cbd 100644 (file)
--- a/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
+++ b/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
@@ -338,7 +338,7 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
     video->prevVop->uChan = video->prevVop->yChan + size;
     video->prevVop->vChan = video->prevVop->uChan + (size >> 2);
 #else
-    if (size > INT32_MAX / 3 * 2) {
+    if (size > INT32_MAX / 3) {
         return PV_FALSE;
     }
     video->currVop->yChan = (PIXEL *) oscl_malloc(size * 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/
@@ -368,7 +368,7 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
         {
             oscl_memset(video->prevEnhcVop, 0, sizeof(Vop));
 #ifndef PV_MEMORY_POOL
-            if (size > INT32_MAX / 3 * 2) {
+            if (size > INT32_MAX / 3) {
                 return PV_FALSE;
             }