From 3c9ec07ef2f7a761bf0418b4d806b50ef57c41f3 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 14 Dec 2004 20:17:23 +0000 Subject: [PATCH] quantizer overflow warning + AAN dct fix Originally committed as revision 3754 to svn://svn.ffmpeg.org/ffmpeg/trunk --- libavcodec/mpegvideo.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index 2b0b315c5..0190c91b5 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -164,7 +164,15 @@ static void convert_matrix(DSPContext *dsp, int (*qmat)[64], uint16_t (*qmat16)[ } for(i=intra; i<64; i++){ - while(((8191LL * qmat[qscale][i]) >> shift) > INT_MAX){ + int64_t max= 8191; + if (dsp->fdct == fdct_ifast +#ifndef FAAN_POSTSCALE + || dsp->fdct == ff_faandct +#endif + ) { + max= (8191LL*aanscales[i]) >> 14; + } + while(((max * qmat[qscale][i]) >> shift) > INT_MAX){ shift++; } } -- 2.11.0