if (quality >= 0)
get_quant_quality(c, quality);
if (width != c->width || height != c->height) {
- int buf_size = 24 + height * width * 3 / 2 + AV_LZO_OUTPUT_PADDING;
+ // also reserve space for a possible additional header
++ int buf_size = 24 + height * width * 3 / 2 + FFMAX(AV_LZO_OUTPUT_PADDING, FF_INPUT_BUFFER_PADDING_SIZE);
+ if (buf_size > INT_MAX/8)
+ return -1;
if ((ret = av_image_check_size(height, width, 0, avctx)) < 0)
return ret;
avctx->width = c->width = width;
buf = &buf[12];
buf_size -= 12;
if (comptype == NUV_RTJPEG_IN_LZO || comptype == NUV_LZO) {
- int outlen = c->decomp_size - AV_LZO_OUTPUT_PADDING, inlen = buf_size;
- int outlen = c->decomp_size - FF_INPUT_BUFFER_PADDING_SIZE;
++ int outlen = c->decomp_size - FFMAX(FF_INPUT_BUFFER_PADDING_SIZE, AV_LZO_OUTPUT_PADDING);
+ int inlen = buf_size;
if (av_lzo1x_decode(c->decomp_buf, &outlen, buf, &inlen)) {
av_log(avctx, AV_LOG_ERROR, "error during lzo decompression\n");
return AVERROR_INVALIDDATA;
}
buf = c->decomp_buf;
- buf_size = c->decomp_size - AV_LZO_OUTPUT_PADDING - outlen;
- buf_size = outlen;
++ buf_size = c->decomp_size - FFMAX(FF_INPUT_BUFFER_PADDING_SIZE, AV_LZO_OUTPUT_PADDING) - outlen;
}
if (c->codec_frameheader) {
int w, h, q;