libhb/deinterlace.c

   1 /*
   2  Copyright (C) 2006 Michael Niedermayer <michaelni@gmx.at>
   3
   4  This program is free software; you can redistribute it and/or modify
   5  it under the terms of the GNU General Public License as published by
   6  the Free Software Foundation; either version 2 of the License, or
   7  (at your option) any later version.
   8
   9  This program is distributed in the hope that it will be useful,
  10  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  GNU General Public License for more details.
  13
  14  You should have received a copy of the GNU General Public License
  15  along with this program; if not, write to the Free Software
  16  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17 */
  18
  19 #include "hb.h"
  20 #include "libavcodec/avcodec.h"
  21 #include "mpeg2dec/mpeg2.h"
  22
  23 #define SUPPRESS_AV_LOG
  24
  25 #define YADIF_MODE_DEFAULT     -1
  26 #define YADIF_PARITY_DEFAULT   -1
  27
  28 #define MCDEINT_MODE_DEFAULT   -1
  29 #define MCDEINT_QP_DEFAULT      1
  30
  31 #define ABS(a) ((a) > 0 ? (a) : (-(a)))
  32 #define MIN3(a,b,c) MIN(MIN(a,b),c)
  33 #define MAX3(a,b,c) MAX(MAX(a,b),c)
  34
  35 struct hb_filter_private_s
  36 {
  37     int              pix_fmt;
  38     int              width[3];
  39     int              height[3];
  40
  41     int              yadif_mode;
  42     int              yadif_parity;
  43     int              yadif_ready;
  44
  45     uint8_t        * yadif_ref[4][3];
  46     int              yadif_ref_stride[3];
  47
  48     int              mcdeint_mode;
  49     int              mcdeint_qp;
  50
  51     int              mcdeint_outbuf_size;
  52     uint8_t        * mcdeint_outbuf;
  53     AVCodecContext * mcdeint_avctx_enc;
  54     AVFrame        * mcdeint_frame;
  55     AVFrame        * mcdeint_frame_dec;
  56
  57     AVPicture        pic_in;
  58     AVPicture        pic_out;
  59     hb_buffer_t *    buf_out[2];
  60     hb_buffer_t *    buf_settings;
  61 };
  62
  63 hb_filter_private_t * hb_deinterlace_init( int pix_fmt,
  64                                            int width,
  65                                            int height,
  66                                            char * settings );
  67
  68 int hb_deinterlace_work( hb_buffer_t * buf_in,
  69                          hb_buffer_t ** buf_out,
  70                          int pix_fmt,
  71                          int width,
  72                          int height,
  73                          hb_filter_private_t * pv );
  74
  75 void hb_deinterlace_close( hb_filter_private_t * pv );
  76
  77 hb_filter_object_t hb_filter_deinterlace =
  78 {
  79     FILTER_DEINTERLACE,
  80     "Deinterlace (ffmpeg or yadif/mcdeint)",
  81     NULL,
  82     hb_deinterlace_init,
  83     hb_deinterlace_work,
  84     hb_deinterlace_close,
  85 };
  86
  87 static void yadif_store_ref( const uint8_t ** pic,
  88                              hb_filter_private_t * pv )
  89 {
  90     memcpy( pv->yadif_ref[3],
  91             pv->yadif_ref[0],
  92             sizeof(uint8_t *)*3 );
  93
  94     memmove( pv->yadif_ref[0],
  95              pv->yadif_ref[1],
  96              sizeof(uint8_t *)*3*3 );
  97
  98     int i;
  99     for( i = 0; i < 3; i++ )
 100     {
 101         const uint8_t * src = pic[i];
 102         uint8_t * ref = pv->yadif_ref[2][i];
 103
 104         int w = pv->width[i];
 105         int h = pv->height[i];
 106         int ref_stride = pv->yadif_ref_stride[i];
 107
 108         int y;
 109         for( y = 0; y < pv->height[i]; y++ )
 110         {
 111             memcpy(ref, src, w);
 112             src = (uint8_t*)src + w;
 113             ref = (uint8_t*)ref + ref_stride;
 114         }
 115     }
 116 }
 117
 118 static void yadif_filter_line( uint8_t *dst,
 119                                uint8_t *prev,
 120                                uint8_t *cur,
 121                                uint8_t *next,
 122                                int plane,
 123                                int parity,
 124                                hb_filter_private_t * pv )
 125 {
 126     uint8_t *prev2 = parity ? prev : cur ;
 127     uint8_t *next2 = parity ? cur  : next;
 128
 129     int w = pv->width[plane];
 130     int refs = pv->yadif_ref_stride[plane];
 131
 132     int x;
 133     for( x = 0; x < w; x++)
 134     {
 135         int c              = cur[-refs];
 136         int d              = (prev2[0] + next2[0])>>1;
 137         int e              = cur[+refs];
 138         int temporal_diff0 = ABS(prev2[0] - next2[0]);
 139         int temporal_diff1 = ( ABS(prev[-refs] - c) + ABS(prev[+refs] - e) ) >> 1;
 140         int temporal_diff2 = ( ABS(next[-refs] - c) + ABS(next[+refs] - e) ) >> 1;
 141         int diff           = MAX3(temporal_diff0>>1, temporal_diff1, temporal_diff2);
 142         int spatial_pred   = (c+e)>>1;
 143         int spatial_score  = ABS(cur[-refs-1] - cur[+refs-1]) + ABS(c-e) +
 144                              ABS(cur[-refs+1] - cur[+refs+1]) - 1;
 145
 146 #define YADIF_CHECK(j)\
 147         {   int score = ABS(cur[-refs-1+j] - cur[+refs-1-j])\
 148                       + ABS(cur[-refs  +j] - cur[+refs  -j])\
 149                       + ABS(cur[-refs+1+j] - cur[+refs+1-j]);\
 150             if( score < spatial_score ){\
 151                 spatial_score = score;\
 152                 spatial_pred  = (cur[-refs  +j] + cur[+refs  -j])>>1;\
 153
 154         YADIF_CHECK(-1) YADIF_CHECK(-2) }} }}
 155         YADIF_CHECK( 1) YADIF_CHECK( 2) }} }}
 156
 157         if( pv->yadif_mode < 2 )
 158         {
 159             int b = (prev2[-2*refs] + next2[-2*refs])>>1;
 160             int f = (prev2[+2*refs] + next2[+2*refs])>>1;
 161
 162             int max = MAX3(d-e, d-c, MIN(b-c, f-e));
 163             int min = MIN3(d-e, d-c, MAX(b-c, f-e));
 164
 165             diff = MAX3( diff, min, -max );
 166         }
 167
 168         if( spatial_pred > d + diff )
 169         {
 170             spatial_pred = d + diff;
 171         }
 172         else if( spatial_pred < d - diff )
 173         {
 174             spatial_pred = d - diff;
 175         }
 176
 177         dst[0] = spatial_pred;
 178
 179         dst++;
 180         cur++;
 181         prev++;
 182         next++;
 183         prev2++;
 184         next2++;
 185     }
 186 }
 187
 188 static void yadif_filter( uint8_t ** dst,
 189                           int parity,
 190                           int tff,
 191                           hb_filter_private_t * pv )
 192 {
 193     int i;
 194     for( i = 0; i < 3; i++ )
 195     {
 196         int w = pv->width[i];
 197         int h = pv->height[i];
 198         int ref_stride = pv->yadif_ref_stride[i];
 199
 200         int y;
 201         for( y = 0; y < h; y++ )
 202         {
 203             if( (y ^ parity) &  1 )
 204             {
 205                 uint8_t *prev = &pv->yadif_ref[0][i][y*ref_stride];
 206                 uint8_t *cur  = &pv->yadif_ref[1][i][y*ref_stride];
 207                 uint8_t *next = &pv->yadif_ref[2][i][y*ref_stride];
 208                 uint8_t *dst2 = &dst[i][y*w];
 209
 210                 yadif_filter_line( dst2, prev, cur, next, i, parity ^ tff, pv );
 211             }
 212             else
 213             {
 214                 memcpy( &dst[i][y*w],
 215                         &pv->yadif_ref[1][i][y*ref_stride],
 216                         w * sizeof(uint8_t) );
 217             }
 218         }
 219     }
 220 }
 221
 222 static void mcdeint_filter( uint8_t ** dst,
 223                             uint8_t ** src,
 224                             int parity,
 225                             hb_filter_private_t * pv )
 226 {
 227     int x, y, i;
 228     int out_size;
 229
 230 #ifdef SUPPRESS_AV_LOG
 231     /* TODO: temporarily change log level to suppress obnoxious debug output */
 232     int loglevel = av_log_get_level();
 233     av_log_set_level( AV_LOG_QUIET );
 234 #endif
 235
 236     for( i=0; i<3; i++ )
 237     {
 238         pv->mcdeint_frame->data[i] = src[i];
 239         pv->mcdeint_frame->linesize[i] = pv->width[i];
 240     }
 241     pv->mcdeint_avctx_enc->me_cmp     = FF_CMP_SAD;
 242     pv->mcdeint_avctx_enc->me_sub_cmp = FF_CMP_SAD;
 243     pv->mcdeint_frame->quality        = pv->mcdeint_qp * FF_QP2LAMBDA;
 244
 245     out_size = avcodec_encode_video( pv->mcdeint_avctx_enc,
 246                                      pv->mcdeint_outbuf,
 247                                      pv->mcdeint_outbuf_size,
 248                                      pv->mcdeint_frame );
 249
 250     pv->mcdeint_frame_dec = pv->mcdeint_avctx_enc->coded_frame;
 251
 252     for( i = 0; i < 3; i++ )
 253     {
 254         int w    = pv->width[i];
 255         int h    = pv->height[i];
 256         int fils = pv->mcdeint_frame_dec->linesize[i];
 257         int srcs = pv->width[i];
 258
 259         for( y = 0; y < h; y++ )
 260         {
 261             if( (y ^ parity) & 1 )
 262             {
 263                 for( x = 0; x < w; x++ )
 264                 {
 265                     if( (x-2)+(y-1)*w >= 0 && (x+2)+(y+1)*w < w*h )
 266                     {
 267                         uint8_t * filp =
 268                             &pv->mcdeint_frame_dec->data[i][x + y*fils];
 269                         uint8_t * srcp = &src[i][x + y*srcs];
 270
 271                         int diff0 = filp[-fils] - srcp[-srcs];
 272                         int diff1 = filp[+fils] - srcp[+srcs];
 273
 274                         int spatial_score =
 275                               ABS(srcp[-srcs-1] - srcp[+srcs-1])
 276                             + ABS(srcp[-srcs  ] - srcp[+srcs  ])
 277                             + ABS(srcp[-srcs+1] - srcp[+srcs+1]) - 1;
 278
 279                         int temp = filp[0];
 280
 281 #define MCDEINT_CHECK(j)\
 282                         {   int score = ABS(srcp[-srcs-1+j] - srcp[+srcs-1-j])\
 283                                       + ABS(srcp[-srcs  +j] - srcp[+srcs  -j])\
 284                                       + ABS(srcp[-srcs+1+j] - srcp[+srcs+1-j]);\
 285                             if( score < spatial_score ) {\
 286                                 spatial_score = score;\
 287                                 diff0 = filp[-fils+j] - srcp[-srcs+j];\
 288                                 diff1 = filp[+fils-j] - srcp[+srcs-j];
 289
 290                         MCDEINT_CHECK(-1) MCDEINT_CHECK(-2) }} }}
 291                         MCDEINT_CHECK( 1) MCDEINT_CHECK( 2) }} }}
 292
 293                         if(diff0 + diff1 > 0)
 294                         {
 295                             temp -= (diff0 + diff1 -
 296                                      ABS( ABS(diff0) - ABS(diff1) ) / 2) / 2;
 297                         }
 298                         else
 299                         {
 300                             temp -= (diff0 + diff1 +
 301                                      ABS( ABS(diff0) - ABS(diff1) ) / 2) / 2;
 302                         }
 303
 304                         filp[0] = dst[i][x + y*w] =
 305                             temp > 255U ? ~(temp>>31) : temp;
 306                     }
 307                     else
 308                     {
 309                         dst[i][x + y*w] =
 310                             pv->mcdeint_frame_dec->data[i][x + y*fils];
 311                     }
 312                 }
 313             }
 314         }
 315
 316         for( y = 0; y < h; y++ )
 317         {
 318             if( !((y ^ parity) & 1) )
 319             {
 320                 for( x = 0; x < w; x++ )
 321                 {
 322                     pv->mcdeint_frame_dec->data[i][x + y*fils] =
 323                         dst[i][x + y*w]= src[i][x + y*srcs];
 324                 }
 325             }
 326         }
 327     }
 328
 329 #ifdef SUPPRESS_AV_LOG
 330     /* TODO: restore previous log level */
 331     av_log_set_level(loglevel);
 332 #endif
 333 }
 334
 335 hb_filter_private_t * hb_deinterlace_init( int pix_fmt,
 336                                            int width,
 337                                            int height,
 338                                            char * settings )
 339 {
 340     if( pix_fmt != PIX_FMT_YUV420P )
 341     {
 342         return 0;
 343     }
 344
 345     hb_filter_private_t * pv = calloc( 1, sizeof(struct hb_filter_private_s) );
 346
 347     pv->pix_fmt = pix_fmt;
 348
 349     pv->width[0]  = width;
 350     pv->height[0] = height;
 351     pv->width[1]  = pv->width[2]  = width >> 1;
 352     pv->height[1] = pv->height[2] = height >> 1;
 353
 354     int buf_size = 3 * width * height / 2;
 355     pv->buf_out[0] = hb_buffer_init( buf_size );
 356     pv->buf_out[1] = hb_buffer_init( buf_size );
 357     pv->buf_settings = hb_buffer_init( 0 );
 358
 359     pv->yadif_ready    = 0;
 360     pv->yadif_mode     = YADIF_MODE_DEFAULT;
 361     pv->yadif_parity   = YADIF_PARITY_DEFAULT;
 362
 363     pv->mcdeint_mode   = MCDEINT_MODE_DEFAULT;
 364     pv->mcdeint_qp     = MCDEINT_QP_DEFAULT;
 365
 366     if( settings )
 367     {
 368         sscanf( settings, "%d:%d:%d:%d",
 369                 &pv->yadif_mode,
 370                 &pv->yadif_parity,
 371                 &pv->mcdeint_mode,
 372                 &pv->mcdeint_qp );
 373     }
 374
 375     /* Allocate yadif specific buffers */
 376     if( pv->yadif_mode >= 0 )
 377     {
 378         int i, j;
 379         for( i = 0; i < 3; i++ )
 380         {
 381             int is_chroma = !!i;
 382             int w = ((width   + 31) & (~31))>>is_chroma;
 383             int h = ((height+6+ 31) & (~31))>>is_chroma;
 384
 385             pv->yadif_ref_stride[i] = w;
 386
 387             for( j = 0; j < 3; j++ )
 388             {
 389                 pv->yadif_ref[j][i] = malloc( w*h*sizeof(uint8_t) ) + 3*w;
 390             }
 391         }
 392     }
 393
 394     /* Allocate mcdeint specific buffers */
 395     if( pv->mcdeint_mode >= 0 )
 396     {
 397         avcodec_init();
 398         avcodec_register_all();
 399
 400         AVCodec * enc = avcodec_find_encoder( CODEC_ID_SNOW );
 401
 402         int i;
 403         for (i = 0; i < 3; i++ )
 404         {
 405             AVCodecContext * avctx_enc;
 406
 407             avctx_enc = pv->mcdeint_avctx_enc = avcodec_alloc_context();
 408
 409             avctx_enc->width                    = width;
 410             avctx_enc->height                   = height;
 411             avctx_enc->time_base                = (AVRational){1,25};  // meaningless
 412             avctx_enc->gop_size                 = 300;
 413             avctx_enc->max_b_frames             = 0;
 414             avctx_enc->pix_fmt                  = PIX_FMT_YUV420P;
 415             avctx_enc->flags                    = CODEC_FLAG_QSCALE | CODEC_FLAG_LOW_DELAY;
 416             avctx_enc->strict_std_compliance    = FF_COMPLIANCE_EXPERIMENTAL;
 417             avctx_enc->global_quality           = 1;
 418             avctx_enc->flags2                   = CODEC_FLAG2_MEMC_ONLY;
 419             avctx_enc->me_cmp                   = FF_CMP_SAD; //SSE;
 420             avctx_enc->me_sub_cmp               = FF_CMP_SAD; //SSE;
 421             avctx_enc->mb_cmp                   = FF_CMP_SSE;
 422
 423             switch( pv->mcdeint_mode )
 424             {
 425                 case 3:
 426                     avctx_enc->refs = 3;
 427                 case 2:
 428                     avctx_enc->me_method = ME_UMH;
 429                 case 1:
 430                     avctx_enc->flags |= CODEC_FLAG_4MV;
 431                     avctx_enc->dia_size =2;
 432                 case 0:
 433                     avctx_enc->flags |= CODEC_FLAG_QPEL;
 434             }
 435
 436             avcodec_open(avctx_enc, enc);
 437         }
 438
 439         pv->mcdeint_frame       = avcodec_alloc_frame();
 440         pv->mcdeint_outbuf_size = width * height * 10;
 441         pv->mcdeint_outbuf      = malloc( pv->mcdeint_outbuf_size );
 442     }
 443
 444     return pv;
 445 }
 446
 447 void hb_deinterlace_close( hb_filter_private_t * pv )
 448 {
 449     if( !pv )
 450     {
 451         return;
 452     }
 453
 454     /* Cleanup frame buffers */
 455     if( pv->buf_out[0] )
 456     {
 457         hb_buffer_close( &pv->buf_out[0] );
 458     }
 459     if( pv->buf_out[1] )
 460     {
 461         hb_buffer_close( &pv->buf_out[1] );
 462     }
 463     if (pv->buf_settings )
 464     {
 465         hb_buffer_close( &pv->buf_settings );
 466     }
 467
 468     /* Cleanup yadif specific buffers */
 469     if( pv->yadif_mode >= 0 )
 470     {
 471         int i;
 472         for( i = 0; i<3*3; i++ )
 473         {
 474             uint8_t **p = &pv->yadif_ref[i%3][i/3];
 475             if (*p)
 476             {
 477                 free( *p - 3*pv->yadif_ref_stride[i/3] );
 478                 *p = NULL;
 479             }
 480         }
 481     }
 482
 483     /* Cleanup mcdeint specific buffers */
 484     if( pv->mcdeint_mode >= 0 )
 485     {
 486         if( pv->mcdeint_avctx_enc )
 487         {
 488             avcodec_close( pv->mcdeint_avctx_enc );
 489             av_freep( &pv->mcdeint_avctx_enc );
 490         }
 491         if( pv->mcdeint_outbuf )
 492         {
 493             free( pv->mcdeint_outbuf );
 494         }
 495     }
 496
 497     free( pv );
 498 }
 499
 500 int hb_deinterlace_work( hb_buffer_t * buf_in,
 501                          hb_buffer_t ** buf_out,
 502                          int pix_fmt,
 503                          int width,
 504                          int height,
 505                          hb_filter_private_t * pv )
 506 {
 507     if( !pv ||
 508         pix_fmt != pv->pix_fmt ||
 509         width   != pv->width[0] ||
 510         height  != pv->height[0] )
 511     {
 512         return FILTER_FAILED;
 513     }
 514
 515     avpicture_fill( &pv->pic_in, buf_in->data,
 516                     pix_fmt, width, height );
 517
 518     /* Use libavcodec deinterlace if yadif_mode < 0 */
 519     if( pv->yadif_mode < 0 )
 520     {
 521         avpicture_fill( &pv->pic_out, pv->buf_out[0]->data,
 522                         pix_fmt, width, height );
 523
 524         avpicture_deinterlace( &pv->pic_out, &pv->pic_in,
 525                                pix_fmt, width, height );
 526
 527         hb_buffer_copy_settings( pv->buf_out[0], buf_in );
 528
 529         *buf_out = pv->buf_out[0];
 530
 531         return FILTER_OK;
 532     }
 533
 534     /* Determine if top-field first layout */
 535     int tff;
 536     if( pv->yadif_parity < 0 )
 537     {
 538         tff = !!(buf_in->flags & PIC_FLAG_TOP_FIELD_FIRST);
 539     }
 540     else
 541     {
 542         tff = (pv->yadif_parity & 1) ^ 1;
 543     }
 544
 545     /* Store current frame in yadif cache */
 546     yadif_store_ref( (const uint8_t**)pv->pic_in.data, pv );
 547
 548     /* If yadif is not ready, store another ref and return FILTER_DELAY */
 549     if( pv->yadif_ready == 0 )
 550     {
 551         yadif_store_ref( (const uint8_t**)pv->pic_in.data, pv );
 552
 553         hb_buffer_copy_settings( pv->buf_settings, buf_in );
 554
 555         /* don't let 'work_loop' send a chapter mark upstream */
 556         buf_in->new_chap  = 0;
 557
 558         pv->yadif_ready = 1;
 559
 560         return FILTER_DELAY;
 561     }
 562
 563     /* Perform yadif and mcdeint filtering */
 564     int frame;
 565     for( frame = 0; frame <= (pv->yadif_mode & 1); frame++ )
 566     {
 567         int parity = frame ^ tff ^ 1;
 568
 569         avpicture_fill( &pv->pic_out, pv->buf_out[!(frame^1)]->data,
 570                         pix_fmt, width, height );
 571
 572         yadif_filter( pv->pic_out.data, parity, tff, pv );
 573
 574         if( pv->mcdeint_mode >= 0 )
 575         {
 576             avpicture_fill( &pv->pic_in,  pv->buf_out[(frame^1)]->data,
 577                             pix_fmt, width, height );
 578
 579             mcdeint_filter( pv->pic_in.data, pv->pic_out.data, parity, pv );
 580
 581             *buf_out = pv->buf_out[ (frame^1)];
 582         }
 583         else
 584         {
 585             *buf_out = pv->buf_out[!(frame^1)];
 586         }
 587     }
 588
 589     /* Copy buffered settings to output buffer settings */
 590     hb_buffer_copy_settings( *buf_out, pv->buf_settings );
 591
 592     /* Replace buffered settings with input buffer settings */
 593     hb_buffer_copy_settings( pv->buf_settings, buf_in );
 594
 595     /* don't let 'work_loop' send a chapter mark upstream */
 596     buf_in->new_chap  = 0;
 597
 598     return FILTER_OK;
 599 }
 600
 601