Merge commit '3beb9cbad35218ed1fb3473eeb3cfc97a931bff4'

* commit '3beb9cbad35218ed1fb3473eeb3cfc97a931bff4':
  roqvideo: check memory allocations and propagate errors

Conflicts:
	libavcodec/roqvideoenc.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

diff --cc libavcodec/roqvideoenc.c
index 694792e,fb7635a..2f8e122
--- 1/libavcodec/roqvideoenc.c
--- 2/libavcodec/roqvideoenc.c
+++ b/libavcodec/roqvideoenc.c
@@@ -249,7 -248,9 +249,9 @@@ static int create_cel_evals(RoqContext 
  {
      int n=0, x, y, i;
  
 -    tempData->cel_evals = av_malloc(enc->width*enc->height/64 * sizeof(CelEvaluation));
 +    tempData->cel_evals = av_malloc_array(enc->width*enc->height/64, sizeof(CelEvaluation));
+     if (!tempData->cel_evals)
+         return AVERROR(ENOMEM);
  
      /* Map to the ROQ quadtree order */
      for (y=0; y<enc->height; y+=16)
@@@ -792,26 -795,36 +796,36 @@@ static void create_clusters(const AVFra
          }
  }
  
- static void generate_codebook(RoqContext *enc, RoqTempdata *tempdata,
-                               int *points, int inputCount, roq_cell *results,
-                               int size, int cbsize)
+ static int generate_codebook(RoqContext *enc, RoqTempdata *tempdata,
+                              int *points, int inputCount, roq_cell *results,
+                              int size, int cbsize)
  {
-     int i, j, k;
+     int i, j, k, ret = 0;
      int c_size = size*size/4;
      int *buf;
 -    int *codebook = av_malloc(6*c_size*cbsize*sizeof(int));
 +    int *codebook = av_malloc_array(6*c_size, cbsize*sizeof(int));
      int *closest_cb;
  
-     if (size == 4)
+     if (!codebook)
+         return AVERROR(ENOMEM);
+ 
+     if (size == 4) {
 -        closest_cb = av_malloc(6*c_size*inputCount*sizeof(int));
 +        closest_cb = av_malloc_array(6*c_size, inputCount*sizeof(int));
-     else
+         if (!closest_cb) {
+             ret = AVERROR(ENOMEM);
+             goto out;
+         }
+     } else
          closest_cb = tempdata->closest_cb2;
  
-     avpriv_init_elbg(points, 6*c_size, inputCount, codebook, cbsize, 1, closest_cb, &enc->randctx);
-     avpriv_do_elbg(points, 6*c_size, inputCount, codebook, cbsize, 1, closest_cb, &enc->randctx);
- 
-     if (size == 4)
-         av_free(closest_cb);
 -    ret = ff_init_elbg(points, 6 * c_size, inputCount, codebook,
++    ret = avpriv_init_elbg(points, 6 * c_size, inputCount, codebook,
+                        cbsize, 1, closest_cb, &enc->randctx);
+     if (ret < 0)
+         goto out;
 -    ret = ff_do_elbg(points, 6 * c_size, inputCount, codebook,
++    ret = avpriv_do_elbg(points, 6 * c_size, inputCount, codebook,
+                      cbsize, 1, closest_cb, &enc->randctx);
+     if (ret < 0)
+         goto out;
  
      buf = codebook;
      for (i=0; i<cbsize; i++)
@@@ -834,10 -850,15 +851,15 @@@ static int generate_new_codebooks(RoqCo
      int max = enc->width*enc->height/16;
      uint8_t mb2[3*4];
      roq_cell *results4 = av_malloc(sizeof(roq_cell)*MAX_CBS_4x4*4);
 -    uint8_t *yuvClusters=av_malloc(sizeof(int)*max*6*4);
 -    int *points = av_malloc(max*6*4*sizeof(int));
 +    uint8_t *yuvClusters=av_malloc_array(max, sizeof(int)*6*4);
 +    int *points = av_malloc_array(max, 6*4*sizeof(int));
      int bias;
  
+     if (!results4 || !yuvClusters || !points) {
+         ret = AVERROR(ENOMEM);
+         goto out;
+     }
+ 
      /* Subsample YUV data */
      create_clusters(enc->frame_to_enc, enc->width, enc->height, yuvClusters);
  
@@@ -848,14 -869,22 +870,22 @@@
      }
  
      /* Create 4x4 codebooks */
-     generate_codebook(enc, tempData, points, max, results4, 4, (enc->quake3_compat ? MAX_CBS_4x4-1 : MAX_CBS_4x4));
+     if ((ret = generate_codebook(enc, tempData, points, max,
 -                                 results4, 4, MAX_CBS_4x4)) < 0)
++                                 results4, 4, (enc->quake3_compat ? MAX_CBS_4x4-1 : MAX_CBS_4x4))) < 0)
+         goto out;
  
 -    codebooks->numCB4 = MAX_CBS_4x4;
 +    codebooks->numCB4 = (enc->quake3_compat ? MAX_CBS_4x4-1 : MAX_CBS_4x4);
  
 -    tempData->closest_cb2 = av_malloc(max*4*sizeof(int));
 +    tempData->closest_cb2 = av_malloc_array(max, 4*sizeof(int));
+     if (!tempData->closest_cb2) {
+         ret = AVERROR(ENOMEM);
+         goto out;
+     }
  
      /* Create 2x2 codebooks */
-     generate_codebook(enc, tempData, points, max*4, enc->cb2x2, 2, MAX_CBS_2x2);
+     if ((ret = generate_codebook(enc, tempData, points, max * 4,
+                                  enc->cb2x2, 2, MAX_CBS_2x2)) < 0)
+         goto out;
  
      codebooks->numCB2 = MAX_CBS_2x2;