smackerdemuxer: check some values before instead of just after malloc()

Fixes Ticket777
Bug Found by: Diana Elena Muscalu

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavformat/smacker.c b/libavformat/smacker.c
index b3de8c7..26319d4 100644 (file)
--- a/libavformat/smacker.c
+++ b/libavformat/smacker.c
@@ -255,6 +255,8 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
             memcpy(oldpal, pal, 768);
             size = avio_r8(s->pb);
             size = size * 4 - 1;
+            if(size + 1 > frame_size)
+                return AVERROR_INVALIDDATA;
             frame_size -= size;
             frame_size--;
             sz = 0;
@@ -289,10 +291,12 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
         /* if audio chunks are present, put them to stack and retrieve later */
         for(i = 0; i < 7; i++) {
             if(flags & 1) {
-                int size;
+                unsigned int size;
                 uint8_t *tmpbuf;
 
                 size = avio_rl32(s->pb) - 4;
+                if(size + 4L > frame_size)
+                    return AVERROR_INVALIDDATA;
                 frame_size -= size;
                 frame_size -= 4;
                 smk->curstream++;