avcodec/asvenc: Fix integer overflow in level

Warn if the qscale is too low for the input data and clip levels to
minimize artifacts

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavcodec/asvenc.c b/libavcodec/asvenc.c
index bbf4494..ccd8182 100644 (file)
--- a/libavcodec/asvenc.c
+++ b/libavcodec/asvenc.c
@@ -50,7 +50,7 @@ static inline void asv1_put_level(PutBitContext *pb, int level)
     }
 }
 
-static inline void asv2_put_level(PutBitContext *pb, int level)
+static inline void asv2_put_level(ASV1Context *a, PutBitContext *pb, int level)
 {
     unsigned int index = level + 31;
 
@@ -58,6 +58,10 @@ static inline void asv2_put_level(PutBitContext *pb, int level)
         put_bits(pb, ff_asv2_level_tab[index][1], ff_asv2_level_tab[index][0]);
     } else {
         put_bits(pb, ff_asv2_level_tab[31][1], ff_asv2_level_tab[31][0]);
+        if (level < -128 || level > 127) {
+            av_log(a->avctx, AV_LOG_WARNING, "Cliping level %d, increase qscale\n", level);
+            level = av_clip_int8(level);
+        }
         asv2_put_bits(pb, 8, level & 0xFF);
     }
 }
@@ -150,13 +154,13 @@ static inline void asv2_encode_block(ASV1Context *a, int16_t block[64])
 
         if (ccp) {
             if (ccp & 8)
-                asv2_put_level(&a->pb, block[index + 0]);
+                asv2_put_level(a, &a->pb, block[index + 0]);
             if (ccp & 4)
-                asv2_put_level(&a->pb, block[index + 8]);
+                asv2_put_level(a, &a->pb, block[index + 8]);
             if (ccp & 2)
-                asv2_put_level(&a->pb, block[index + 1]);
+                asv2_put_level(a, &a->pb, block[index + 1]);
             if (ccp & 1)
-                asv2_put_level(&a->pb, block[index + 9]);
+                asv2_put_level(a, &a->pb, block[index + 9]);
         }
     }
 }