libvpx: check if CQ level is in correct bounds

Check whether CQ level is within qmin..qmax and abort on error. If this isn't properly
set, encoding will fail at the first frame.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavcodec/libvpxenc.c b/libavcodec/libvpxenc.c
index ed038ab..04ea031 100644 (file)
--- a/libavcodec/libvpxenc.c
+++ b/libavcodec/libvpxenc.c
@@ -289,6 +289,16 @@ static av_cold int vpx_init(AVCodecContext *avctx,
         enccfg.rc_min_quantizer = avctx->qmin;
     if (avctx->qmax > 0)
         enccfg.rc_max_quantizer = avctx->qmax;
+
+    if (enccfg.rc_end_usage == VPX_CQ) {
+        if (ctx->crf < enccfg.rc_min_quantizer || ctx->crf > enccfg.rc_max_quantizer) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "CQ level must be between minimum and maximum quantizer value (%d-%d)\n",
+                       enccfg.rc_min_quantizer, enccfg.rc_max_quantizer);
+                return AVERROR(EINVAL);
+        }
+    }
+
     enccfg.rc_dropframe_thresh = avctx->frame_skip_threshold;
 
     //0-100 (0 => CBR, 100 => VBR)