h264: dont mess with chroma planes for grayscale h264.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 47331ae..b504dcf 100644 (file)
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -2179,8 +2179,10 @@ static av_always_inline void hl_decode_mb_internal(H264Context *h, int simple,
                                uvlinesize, 1, 0, simple, pixel_shift);
 
             if (simple || !CONFIG_GRAY || !(s->flags & CODEC_FLAG_GRAY)) {
-                h->hpc.pred8x8[h->chroma_pred_mode](dest_cb, uvlinesize);
-                h->hpc.pred8x8[h->chroma_pred_mode](dest_cr, uvlinesize);
+                if (CHROMA) {
+                    h->hpc.pred8x8[h->chroma_pred_mode](dest_cb, uvlinesize);
+                    h->hpc.pred8x8[h->chroma_pred_mode](dest_cr, uvlinesize);
+                }
             }
 
             hl_decode_mb_predict_luma(h, mb_type, is_h264, simple,

diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index d0dfca3..7739754 100644 (file)
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -87,6 +87,7 @@
 #define CABAC h->pps.cabac
 #endif
 
+#define CHROMA    (h->sps.chroma_format_idc)
 #define CHROMA422 (h->sps.chroma_format_idc == 2)
 #define CHROMA444 (h->sps.chroma_format_idc == 3)
 

diff --git a/libavcodec/h264_loopfilter.c b/libavcodec/h264_loopfilter.c
index f3a5ff6..6395607 100644 (file)
--- a/libavcodec/h264_loopfilter.c
+++ b/libavcodec/h264_loopfilter.c
@@ -245,7 +245,7 @@ static av_always_inline void h264_filter_mb_fast_internal(H264Context *h,
                                                           int pixel_shift)
 {
     MpegEncContext * const s = &h->s;
-    int chroma = !(CONFIG_GRAY && (s->flags&CODEC_FLAG_GRAY));
+    int chroma = CHROMA && !(CONFIG_GRAY && (s->flags&CODEC_FLAG_GRAY));
     int chroma444 = CHROMA444;
     int chroma422 = CHROMA422;
 
@@ -713,7 +713,7 @@ void ff_h264_filter_mb( H264Context *h, int mb_x, int mb_y, uint8_t *img_y, uint
     const int mvy_limit = IS_INTERLACED(mb_type) ? 2 : 4;
     int first_vertical_edge_done = 0;
     av_unused int dir;
-    int chroma = !(CONFIG_GRAY && (s->flags&CODEC_FLAG_GRAY));
+    int chroma = CHROMA && !(CONFIG_GRAY && (s->flags&CODEC_FLAG_GRAY));
     int qp_bd_offset = 6 * (h->sps.bit_depth_luma - 8);
     int a = h->slice_alpha_c0_offset - qp_bd_offset;
     int b = h->slice_beta_offset - qp_bd_offset;

diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c
index 8b11241..2e9fad0 100644 (file)
--- a/libavcodec/svq3.c
+++ b/libavcodec/svq3.c
@@ -823,6 +823,7 @@ static av_cold int svq3_decode_init(AVCodecContext *avctx)
     s->flags2 = avctx->flags2;
     s->unrestricted_mv = 1;
     h->is_complex=1;
+    h->sps.chroma_format_idc = 1;
     avctx->pix_fmt = avctx->codec->pix_fmts[0];
 
     if (!s->context_initialized) {