libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.fr/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8 #include "hbffmpeg.h"
   9 #include <stdio.h>
  10 #include "samplerate.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_audio_t * audio;
  22
  23     int64_t      next_start;    /* start time of next output frame */
  24     int64_t      next_pts;      /* start time of next input frame */
  25     int64_t      first_drop;    /* PTS of first 'went backwards' frame dropped */
  26     int          drop_count;    /* count of 'time went backwards' drops */
  27
  28     /* Raw */
  29     SRC_STATE  * state;
  30     SRC_DATA     data;
  31
  32     /* AC-3 */
  33     int          ac3_size;
  34     uint8_t    * ac3_buf;
  35
  36 } hb_sync_audio_t;
  37
  38 struct hb_work_private_s
  39 {
  40     hb_job_t * job;
  41     int        busy;            // bitmask with one bit for each active input
  42                                 // (bit 0 = video; 1 = audio 0, 2 = audio 1, ...
  43                                 // appropriate bit is cleared when input gets
  44                                 // an eof buf. syncWork returns done when all
  45                                 // bits are clear.
  46     /* Video */
  47     int64_t pts_offset;
  48     int64_t next_start;         /* start time of next output frame */
  49     int64_t next_pts;           /* start time of next input frame */
  50     int64_t first_drop;         /* PTS of first 'went backwards' frame dropped */
  51     int drop_count;             /* count of 'time went backwards' drops */
  52     int drops;                  /* frames dropped to make a cbr video stream */
  53     int dups;                   /* frames duplicated to make a cbr video stream */
  54     int video_sequence;
  55     int count_frames;
  56     int count_frames_max;
  57     int chap_mark;              /* to propagate chapter mark across a drop */
  58     hb_buffer_t * cur; /* The next picture to process */
  59
  60     /* Audio */
  61     hb_sync_audio_t sync_audio[8];
  62     int64_t audio_passthru_slip;
  63     int64_t video_pts_slip;
  64
  65     /* Statistics */
  66     uint64_t st_counts[4];
  67     uint64_t st_dates[4];
  68     uint64_t st_first;
  69 };
  70
  71 /***********************************************************************
  72  * Local prototypes
  73  **********************************************************************/
  74 static void InitAudio( hb_work_object_t * w, int i );
  75 static void SyncVideo( hb_work_object_t * w );
  76 static void SyncAudio( hb_work_object_t * w, int i );
  77 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
  78 static void UpdateState( hb_work_object_t * w );
  79
  80 /***********************************************************************
  81  * hb_work_sync_init
  82  ***********************************************************************
  83  * Initialize the work object
  84  **********************************************************************/
  85 int syncInit( hb_work_object_t * w, hb_job_t * job )
  86 {
  87     hb_title_t       * title = job->title;
  88     hb_chapter_t     * chapter;
  89     int                i;
  90     uint64_t           duration;
  91     hb_work_private_t * pv;
  92
  93     pv = calloc( 1, sizeof( hb_work_private_t ) );
  94     w->private_data = pv;
  95
  96     pv->job            = job;
  97     pv->pts_offset     = INT64_MIN;
  98
  99     if( job->pass == 2 )
 100     {
 101         /* We already have an accurate frame count from pass 1 */
 102         hb_interjob_t * interjob = hb_interjob_get( job->h );
 103         pv->count_frames_max = interjob->frame_count;
 104     }
 105     else
 106     {
 107         /* Calculate how many video frames we are expecting */
 108         if ( job->pts_to_stop )
 109         {
 110             duration = job->pts_to_stop + 90000;
 111         }
 112         else if( job->frame_to_stop )
 113         {
 114             /* Set the duration to a rough estimate */
 115             duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
 116         }
 117         else
 118         {
 119             duration = 0;
 120             for( i = job->chapter_start; i <= job->chapter_end; i++ )
 121             {
 122                 chapter   = hb_list_item( title->list_chapter, i - 1 );
 123                 duration += chapter->duration;
 124             }
 125             duration += 90000;
 126             /* 1 second safety so we're sure we won't miss anything */
 127         }
 128         pv->count_frames_max = duration * title->rate / title->rate_base / 90000;
 129     }
 130
 131     hb_log( "sync: expecting %d video frames", pv->count_frames_max );
 132     pv->busy |= 1;
 133
 134     /* Initialize libsamplerate for every audio track we have */
 135     if ( ! job->indepth_scan )
 136     {
 137         for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
 138         {
 139             pv->busy |= ( 1 << (i + 1) );
 140             InitAudio( w, i );
 141         }
 142     }
 143
 144     return 0;
 145 }
 146
 147 /***********************************************************************
 148  * Close
 149  ***********************************************************************
 150  *
 151  **********************************************************************/
 152 void syncClose( hb_work_object_t * w )
 153 {
 154     hb_work_private_t * pv = w->private_data;
 155     hb_job_t          * job   = pv->job;
 156     hb_title_t        * title = job->title;
 157     hb_audio_t        * audio = NULL;
 158     int i;
 159
 160     if( pv->cur )
 161     {
 162         hb_buffer_close( &pv->cur );
 163     }
 164
 165     hb_log( "sync: got %d frames, %d expected",
 166             pv->count_frames, pv->count_frames_max );
 167
 168     /* save data for second pass */
 169     if( job->pass == 1 )
 170     {
 171         /* Preserve frame count for better accuracy in pass 2 */
 172         hb_interjob_t * interjob = hb_interjob_get( job->h );
 173         interjob->frame_count = pv->count_frames;
 174         interjob->last_job = job->sequence_id;
 175         interjob->total_time = pv->next_start;
 176     }
 177
 178     if (pv->drops || pv->dups )
 179     {
 180         hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
 181     }
 182
 183     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 184     {
 185         audio = hb_list_item( title->list_audio, i );
 186         if( audio->config.out.codec == HB_ACODEC_AC3 )
 187         {
 188             free( pv->sync_audio[i].ac3_buf );
 189         }
 190         else
 191         {
 192             src_delete( pv->sync_audio[i].state );
 193         }
 194     }
 195
 196     free( pv );
 197     w->private_data = NULL;
 198 }
 199
 200 /***********************************************************************
 201  * Work
 202  ***********************************************************************
 203  * The root routine of this work abject
 204  *
 205  * The way this works is that we are syncing the audio to the PTS of
 206  * the last video that we processed. That's why we skip the audio sync
 207  * if we haven't got a valid PTS from the video yet.
 208  *
 209  **********************************************************************/
 210 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
 211               hb_buffer_t ** unused2 )
 212 {
 213     hb_work_private_t * pv = w->private_data;
 214     int i;
 215
 216     if ( pv->busy & 1 )
 217         SyncVideo( w );
 218
 219     for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
 220     {
 221         if ( pv->busy & ( 1 << (i + 1) ) )
 222             SyncAudio( w, i );
 223     }
 224
 225     return ( pv->busy? HB_WORK_OK : HB_WORK_DONE );
 226 }
 227
 228 hb_work_object_t hb_sync =
 229 {
 230     WORK_SYNC,
 231     "Synchronization",
 232     syncInit,
 233     syncWork,
 234     syncClose
 235 };
 236
 237 static void InitAudio( hb_work_object_t * w, int i )
 238 {
 239     hb_work_private_t * pv = w->private_data;
 240     hb_job_t        * job   = pv->job;
 241     hb_title_t      * title = job->title;
 242     hb_sync_audio_t * sync;
 243
 244     sync        = &pv->sync_audio[i];
 245     sync->audio = hb_list_item( title->list_audio, i );
 246
 247     if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
 248     {
 249         /* Have a silent AC-3 frame ready in case we have to fill a
 250            gap */
 251         AVCodec        * codec;
 252         AVCodecContext * c;
 253         short          * zeros;
 254
 255         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 256         c     = avcodec_alloc_context();
 257
 258         c->bit_rate    = sync->audio->config.in.bitrate;
 259         c->sample_rate = sync->audio->config.in.samplerate;
 260         c->channels    = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
 261
 262         if( hb_avcodec_open( c, codec ) < 0 )
 263         {
 264             hb_log( "sync: avcodec_open failed" );
 265             return;
 266         }
 267
 268         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 269                                  sizeof( short ) * c->channels, 1 );
 270         sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
 271                              sync->audio->config.in.samplerate / 8;
 272         sync->ac3_buf  = malloc( sync->ac3_size );
 273
 274         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 275                                   zeros ) != sync->ac3_size )
 276         {
 277             hb_log( "sync: avcodec_encode_audio failed" );
 278         }
 279
 280         free( zeros );
 281         hb_avcodec_close( c );
 282         av_free( c );
 283     }
 284     else
 285     {
 286         /* Initialize libsamplerate */
 287         int error;
 288         sync->state             = src_new( SRC_SINC_MEDIUM_QUALITY, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
 289         sync->data.end_of_input = 0;
 290     }
 291 }
 292
 293 /***********************************************************************
 294  * SyncVideo
 295  ***********************************************************************
 296  *
 297  **********************************************************************/
 298 static void SyncVideo( hb_work_object_t * w )
 299 {
 300     hb_work_private_t * pv = w->private_data;
 301     hb_buffer_t * cur, * next, * sub = NULL;
 302     hb_job_t * job = pv->job;
 303     hb_subtitle_t *subtitle;
 304     int i;
 305     int64_t pts_skip;
 306
 307     if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
 308     {
 309         /* We haven't even got a frame yet */
 310         return;
 311     }
 312     cur = pv->cur;
 313     pts_skip = 0;
 314     if( cur->size == 0 )
 315     {
 316         /* we got an end-of-stream. Feed it downstream & signal that we're done. */
 317         hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 318
 319         /*
 320          * Push through any subtitle EOFs in case they were not synced through.
 321          */
 322         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 323         {
 324             subtitle = hb_list_item( job->list_subtitle, i );
 325             if( subtitle->config.dest == PASSTHRUSUB )
 326             {
 327                 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 328             }
 329         }
 330
 331         pv->busy &=~ 1;
 332         return;
 333     }
 334
 335     /* At this point we have a frame to process. Let's check
 336         1) if we will be able to push into the fifo ahead
 337         2) if the next frame is there already, since we need it to
 338            compute the duration of the current frame*/
 339     while( !hb_fifo_is_full( job->fifo_sync ) &&
 340            ( next = hb_fifo_see( job->fifo_raw ) ) )
 341     {
 342         hb_buffer_t * buf_tmp;
 343
 344         if( next->size == 0 )
 345         {
 346             /* we got an end-of-stream. Feed it downstream & signal that
 347              * we're done. Note that this means we drop the final frame of
 348              * video (we don't know its duration). On DVDs the final frame
 349              * is often strange and dropping it seems to be a good idea. */
 350             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 351
 352             /*
 353              * Push through any subtitle EOFs in case they were not synced through.
 354              */
 355             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 356             {
 357                 subtitle = hb_list_item( job->list_subtitle, i );
 358                 if( subtitle->config.dest == PASSTHRUSUB )
 359                 {
 360                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 361                 }
 362             }
 363             pv->busy &=~ 1;
 364             return;
 365         }
 366         if( pv->pts_offset == INT64_MIN )
 367         {
 368             /* This is our first frame */
 369             pv->pts_offset = 0;
 370             if ( cur->start != 0 )
 371             {
 372                 /*
 373                  * The first pts from a dvd should always be zero but
 374                  * can be non-zero with a transport or program stream since
 375                  * we're not guaranteed to start on an IDR frame. If we get
 376                  * a non-zero initial PTS extend its duration so it behaves
 377                  * as if it started at zero so that our audio timing will
 378                  * be in sync.
 379                  */
 380                 hb_log( "sync: first pts is %"PRId64, cur->start );
 381                 cur->start = 0;
 382             }
 383         }
 384
 385         /*
 386          * since the first frame is always 0 and the upstream reader code
 387          * is taking care of adjusting for pts discontinuities, we just have
 388          * to deal with the next frame's start being in the past. This can
 389          * happen when the PTS is adjusted after data loss but video frame
 390          * reordering causes some frames with the old clock to appear after
 391          * the clock change. This creates frames that overlap in time which
 392          * looks to us like time going backward. The downstream muxing code
 393          * can deal with overlaps of up to a frame time but anything larger
 394          * we handle by dropping frames here.
 395          */
 396         if ( (int64_t)( next->start - pv->video_pts_slip - cur->start ) <= 0 )
 397         {
 398             if ( pv->first_drop == 0 )
 399             {
 400                 pv->first_drop = next->start;
 401             }
 402             ++pv->drop_count;
 403             if (next->start - cur->start > 0)
 404             {
 405                 pts_skip += next->start - cur->start;
 406                 pv->video_pts_slip -= next->start - cur->start;
 407             }
 408             buf_tmp = hb_fifo_get( job->fifo_raw );
 409             if ( buf_tmp->new_chap )
 410             {
 411                 // don't drop a chapter mark when we drop the buffer
 412                 pv->chap_mark = buf_tmp->new_chap;
 413             }
 414             hb_buffer_close( &buf_tmp );
 415             continue;
 416         }
 417         if ( pv->first_drop )
 418         {
 419             hb_log( "sync: video time didn't advance - dropped %d frames "
 420                     "(delta %d ms, current %"PRId64", next %"PRId64", dur %d)",
 421                     pv->drop_count, (int)( cur->start - pv->first_drop ) / 90,
 422                     cur->start, next->start, (int)( next->start - cur->start ) );
 423             pv->first_drop = 0;
 424             pv->drop_count = 0;
 425         }
 426
 427         /*
 428          * Track the video sequence number localy so that we can sync the audio
 429          * to it using the sequence number as well as the PTS.
 430          */
 431         pv->video_sequence = cur->sequence;
 432
 433         /*
 434          * Look for a subtitle for this frame.
 435          *
 436          * If found then it will be tagged onto a video buffer of the correct time and
 437          * sent in to the render pipeline. This only needs to be done for VOBSUBs which
 438          * get rendered, other types of subtitles can just sit in their raw_queue until
 439          * delt with at muxing.
 440          */
 441         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 442         {
 443             subtitle = hb_list_item( job->list_subtitle, i );
 444
 445             /*
 446              * Rewrite timestamps on subtitles that need it (on raw queue).
 447              */
 448             if( subtitle->source == CC608SUB ||
 449                 subtitle->source == CC708SUB ||
 450                 subtitle->source == SRTSUB )
 451             {
 452                 /*
 453                  * Rewrite timestamps on subtitles that came from Closed Captions
 454                  * since they are using the MPEG2 timestamps.
 455                  */
 456                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 457                 {
 458                     /*
 459                      * Rewrite the timestamps as and when the video
 460                      * (cur->start) reaches the same timestamp as a
 461                      * closed caption (sub->start).
 462                      *
 463                      * What about discontinuity boundaries - not delt
 464                      * with here - Van?
 465                      *
 466                      * Bypass the sync fifo altogether.
 467                      */
 468                     if( sub->size <= 0 )
 469                     {
 470                         sub = hb_fifo_get( subtitle->fifo_raw );
 471                         hb_fifo_push( subtitle->fifo_out, sub );
 472                         sub = NULL;
 473                         break;
 474                     } else {
 475                         /*
 476                          * Sync the subtitles to the incoming video, and use
 477                          * the matching converted video timestamp.
 478                          *
 479                          * Note that it doesn't appear that we need to convert
 480                          * timestamps, I guess that they were already correct,
 481                          * so just push them through for rendering.
 482                          *
 483                          */
 484                         if( sub->start < cur->start )
 485                         {
 486                             sub = hb_fifo_get( subtitle->fifo_raw );
 487                             hb_fifo_push( subtitle->fifo_out, sub );
 488                         } else {
 489                             sub = NULL;
 490                             break;
 491                         }
 492                     }
 493                 }
 494             }
 495
 496             if( subtitle->source == VOBSUB )
 497             {
 498                 hb_buffer_t * sub2;
 499                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 500                 {
 501                     if( sub->size == 0 )
 502                     {
 503                         /*
 504                          * EOF, pass it through immediately.
 505                          */
 506                         break;
 507                     }
 508
 509                     /* If two subtitles overlap, make the first one stop
 510                        when the second one starts */
 511                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 512                     if( sub2 && sub->stop > sub2->start )
 513                     {
 514                         sub->stop = sub2->start;
 515                     }
 516
 517                     // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 518                     //       sub, cur->sequence, sub->sequence);
 519
 520                     if( sub->sequence > cur->sequence )
 521                     {
 522                         /*
 523                          * The video is behind where we are, so wait until
 524                          * it catches up to the same reader point on the
 525                          * DVD. Then our PTS should be in the same region
 526                          * as the video.
 527                          */
 528                         sub = NULL;
 529                         break;
 530                     }
 531
 532                     if( sub->stop > cur->start ) {
 533                         /*
 534                          * The stop time is in the future, so fall through
 535                          * and we'll deal with it in the next block of
 536                          * code.
 537                          */
 538
 539                         /*
 540                          * There is a valid subtitle, is it time to display it?
 541                          */
 542                         if( sub->stop > sub->start)
 543                         {
 544                             /*
 545                              * Normal subtitle which ends after it starts,
 546                              * check to see that the current video is between
 547                              * the start and end.
 548                              */
 549                             if( cur->start > sub->start &&
 550                                 cur->start < sub->stop )
 551                             {
 552                                 /*
 553                                 * We should be playing this, so leave the
 554                                 * subtitle in place.
 555                                 *
 556                                 * fall through to display
 557                                 */
 558                                 if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
 559                                 {
 560                                     /*
 561                                      * Subtitle is on for less than three
 562                                      * seconds, extend the time that it is
 563                                      * displayed to make it easier to read.
 564                                      * Make it 3 seconds or until the next
 565                                      * subtitle is displayed.
 566                                      *
 567                                      * This is in response to Indochine which
 568                                      * only displays subs for 1 second -
 569                                      * too fast to read.
 570                                      */
 571                                     sub->stop = sub->start + ( 2 * 90000 );
 572
 573                                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 574
 575                                     if( sub2 && sub->stop > sub2->start )
 576                                     {
 577                                         sub->stop = sub2->start;
 578                                     }
 579                                 }
 580                             }
 581                             else
 582                             {
 583                                 /*
 584                                  * Defer until the play point is within
 585                                  * the subtitle
 586                                  */
 587                                 sub = NULL;
 588                             }
 589                         }
 590                         else
 591                         {
 592                             /*
 593                              * The end of the subtitle is less than the start,
 594                              * this is a sign of a PTS discontinuity.
 595                              */
 596                             if( sub->start > cur->start )
 597                             {
 598                                 /*
 599                                  * we haven't reached the start time yet, or
 600                                  * we have jumped backwards after having
 601                                  * already started this subtitle.
 602                                  */
 603                                 if( cur->start < sub->stop )
 604                                 {
 605                                     /*
 606                                      * We have jumped backwards and so should
 607                                      * continue displaying this subtitle.
 608                                      *
 609                                      * fall through to display.
 610                                      */
 611                                 }
 612                                 else
 613                                 {
 614                                     /*
 615                                      * Defer until the play point is
 616                                      * within the subtitle
 617                                      */
 618                                     sub = NULL;
 619                                 }
 620                             } else {
 621                                 /*
 622                                 * Play this subtitle as the start is
 623                                 * greater than our video point.
 624                                 *
 625                                 * fall through to display/
 626                                 */
 627                             }
 628                         }
 629                         break;
 630                     }
 631                     else
 632                     {
 633
 634                         /*
 635                          * The subtitle is older than this picture, trash it
 636                          */
 637                         sub = hb_fifo_get( subtitle->fifo_raw );
 638                         hb_buffer_close( &sub );
 639                     }
 640                 }
 641
 642                 /* If we have a subtitle for this picture, copy it */
 643                 /* FIXME: we should avoid this memcpy */
 644                 if( sub )
 645                 {
 646                     if( sub->size > 0 )
 647                     {
 648                         if( subtitle->config.dest == RENDERSUB )
 649                         {
 650                             if ( cur->sub == NULL )
 651                             {
 652                                 /*
 653                                  * Tack onto the video buffer for rendering
 654                                  */
 655                                 cur->sub         = hb_buffer_init( sub->size );
 656                                 cur->sub->x      = sub->x;
 657                                 cur->sub->y      = sub->y;
 658                                 cur->sub->width  = sub->width;
 659                                 cur->sub->height = sub->height;
 660                                 memcpy( cur->sub->data, sub->data, sub->size );
 661                             }
 662                         } else {
 663                             /*
 664                              * Pass-Through, pop it off of the raw queue,
 665                              */
 666                             sub = hb_fifo_get( subtitle->fifo_raw );
 667                             hb_fifo_push( subtitle->fifo_sync, sub );
 668                         }
 669                     } else {
 670                         /*
 671                         * EOF - consume for rendered, else pass through
 672                         */
 673                         if( subtitle->config.dest == RENDERSUB )
 674                         {
 675                             sub = hb_fifo_get( subtitle->fifo_raw );
 676                             hb_buffer_close( &sub );
 677                         } else {
 678                             sub = hb_fifo_get( subtitle->fifo_raw );
 679                             hb_fifo_push( subtitle->fifo_out, sub );
 680                         }
 681                     }
 682                 }
 683             }
 684         } // end subtitles
 685
 686         /*
 687          * Adjust the pts of the current frame so that it's contiguous
 688          * with the previous frame. The start time of the current frame
 689          * has to be the end time of the previous frame and the stop
 690          * time has to be the start of the next frame.  We don't
 691          * make any adjustments to the source timestamps other than removing
 692          * the clock offsets (which also removes pts discontinuities).
 693          * This means we automatically encode at the source's frame rate.
 694          * MP2 uses an implicit duration (frames end when the next frame
 695          * starts) but more advanced containers like MP4 use an explicit
 696          * duration. Since we're looking ahead one frame we set the
 697          * explicit stop time from the start time of the next frame.
 698          */
 699         buf_tmp = cur;
 700         pv->cur = cur = hb_fifo_get( job->fifo_raw );
 701         cur->sub = NULL;
 702         pv->next_pts = cur->start;
 703         int64_t duration = cur->start - pts_skip - buf_tmp->start;
 704         pts_skip = 0;
 705         if ( duration <= 0 )
 706         {
 707             hb_log( "sync: invalid video duration %"PRId64", start %"PRId64", next %"PRId64"",
 708                     duration, buf_tmp->start, next->start );
 709         }
 710
 711         buf_tmp->start = pv->next_start;
 712         pv->next_start += duration;
 713         buf_tmp->stop = pv->next_start;
 714
 715         if ( pv->chap_mark )
 716         {
 717             // we have a pending chapter mark from a recent drop - put it on this
 718             // buffer (this may make it one frame late but we can't do any better).
 719             buf_tmp->new_chap = pv->chap_mark;
 720             pv->chap_mark = 0;
 721         }
 722
 723         /* Push the frame to the renderer */
 724         hb_fifo_push( job->fifo_sync, buf_tmp );
 725
 726         /* Update UI */
 727         UpdateState( w );
 728
 729         if( job->frame_to_stop && pv->count_frames > job->frame_to_stop )
 730         {
 731             // Drop an empty buffer into our output to ensure that things
 732             // get flushed all the way out.
 733             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 734             pv->busy &=~ 1;
 735             hb_log( "sync: reached %d frames, exiting early (%i busy)",
 736                     pv->count_frames, pv->busy );
 737             return;
 738         }
 739     }
 740 }
 741
 742 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
 743                               hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
 744 {
 745     int64_t start = sync->next_start;
 746     int64_t duration = buf->stop - buf->start;
 747
 748     sync->next_pts += duration;
 749
 750     if( audio->config.in.samplerate == audio->config.out.samplerate ||
 751         audio->config.out.codec == HB_ACODEC_AC3 ||
 752         audio->config.out.codec == HB_ACODEC_DCA )
 753     {
 754         /*
 755          * If we don't have to do sample rate conversion or this audio is
 756          * pass-thru just send the input buffer downstream after adjusting
 757          * its timestamps to make the output stream continuous.
 758          */
 759     }
 760     else
 761     {
 762         /* Not pass-thru - do sample rate conversion */
 763         int count_in, count_out;
 764         hb_buffer_t * buf_raw = buf;
 765         int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
 766                             sizeof( float );
 767
 768         count_in  = buf_raw->size / channel_count;
 769         /*
 770          * When using stupid rates like 44.1 there will always be some
 771          * truncation error. E.g., a 1536 sample AC3 frame will turn into a
 772          * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
 773          * the error will build up over time and eventually the audio will
 774          * substantially lag the video. libsamplerate will keep track of the
 775          * fractional sample & give it to us when appropriate if we give it
 776          * an extra sample of space in the output buffer.
 777          */
 778         count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
 779
 780         sync->data.input_frames = count_in;
 781         sync->data.output_frames = count_out;
 782         sync->data.src_ratio = (double)audio->config.out.samplerate /
 783                                (double)audio->config.in.samplerate;
 784
 785         buf = hb_buffer_init( count_out * channel_count );
 786         sync->data.data_in  = (float *) buf_raw->data;
 787         sync->data.data_out = (float *) buf->data;
 788         if( src_process( sync->state, &sync->data ) )
 789         {
 790             /* XXX If this happens, we're screwed */
 791             hb_log( "sync: audio %d src_process failed", i );
 792         }
 793         hb_buffer_close( &buf_raw );
 794
 795         buf->size = sync->data.output_frames_gen * channel_count;
 796         duration = ( sync->data.output_frames_gen * 90000 ) /
 797                    audio->config.out.samplerate;
 798     }
 799     buf->frametype = HB_FRAME_AUDIO;
 800     buf->start = start;
 801     buf->stop  = start + duration;
 802     sync->next_start = start + duration;
 803     hb_fifo_push( fifo, buf );
 804 }
 805
 806 /***********************************************************************
 807  * SyncAudio
 808  ***********************************************************************
 809  *
 810  **********************************************************************/
 811 static void SyncAudio( hb_work_object_t * w, int i )
 812 {
 813     hb_work_private_t * pv = w->private_data;
 814     hb_job_t        * job = pv->job;
 815     hb_sync_audio_t * sync = &pv->sync_audio[i];
 816     hb_audio_t      * audio = sync->audio;
 817     hb_buffer_t     * buf;
 818     hb_fifo_t       * fifo;
 819     int64_t start;
 820
 821     if( audio->config.out.codec == HB_ACODEC_AC3 ||
 822         audio->config.out.codec == HB_ACODEC_DCA )
 823     {
 824         fifo = audio->priv.fifo_out;
 825     }
 826     else
 827     {
 828         fifo = audio->priv.fifo_sync;
 829     }
 830
 831     while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
 832     {
 833         start = buf->start - pv->audio_passthru_slip;
 834         /* if the next buffer is an eof send it downstream */
 835         if ( buf->size <= 0 )
 836         {
 837             buf = hb_fifo_get( audio->priv.fifo_raw );
 838             hb_fifo_push( fifo, buf );
 839             pv->busy &=~ (1 << (i + 1) );
 840             return;
 841         }
 842         if( job->frame_to_stop && pv->count_frames >= job->frame_to_stop )
 843         {
 844             hb_fifo_push( fifo, hb_buffer_init(0) );
 845             pv->busy &=~ (1 << (i + 1) );
 846             return;
 847         }
 848         if ( (int64_t)( start - sync->next_pts ) < 0 )
 849         {
 850             // audio time went backwards.
 851             // If our output clock is more than a half frame ahead of the
 852             // input clock drop this frame to move closer to sync.
 853             // Otherwise drop frames until the input clock matches the output clock.
 854             if ( sync->first_drop || sync->next_start - start > 90*15 )
 855             {
 856                 // Discard data that's in the past.
 857                 if ( sync->first_drop == 0 )
 858                 {
 859                     sync->first_drop = sync->next_pts;
 860                 }
 861                 ++sync->drop_count;
 862                 buf = hb_fifo_get( audio->priv.fifo_raw );
 863                 hb_buffer_close( &buf );
 864                 continue;
 865             }
 866             sync->next_pts = start;
 867         }
 868         if ( sync->first_drop )
 869         {
 870             // we were dropping old data but input buf time is now current
 871             hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
 872                     "(next %"PRId64", current %"PRId64")", i,
 873                     (int)( sync->next_pts - sync->first_drop ) / 90,
 874                     sync->drop_count, sync->first_drop, sync->next_pts );
 875             sync->first_drop = 0;
 876             sync->drop_count = 0;
 877             sync->next_pts = start;
 878         }
 879         if ( start - sync->next_pts >= (90 * 70) )
 880         {
 881             if ( start - sync->next_pts > (90000LL * 60) )
 882             {
 883                 // there's a gap of more than a minute between the last
 884                 // frame and this. assume we got a corrupted timestamp
 885                 // and just drop the next buf.
 886                 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
 887                         "  start %"PRId64", next %"PRId64,
 888                         (int)((start - sync->next_pts) / (90000*60)),
 889                         i, start, sync->next_pts );
 890                 buf = hb_fifo_get( audio->priv.fifo_raw );
 891                 hb_buffer_close( &buf );
 892                 continue;
 893             }
 894             /*
 895              * there's a gap of at least 70ms between the last
 896              * frame we processed & the next. Fill it with silence.
 897              * Or in the case of DCA, skip some frames from the
 898              * other streams.
 899              */
 900             if( sync->audio->config.out.codec == HB_ACODEC_DCA )
 901             {
 902                 hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
 903                         "  start %"PRId64", next %"PRId64,
 904                         (int)((start - sync->next_pts) / 90),
 905                         i, start, sync->next_pts );
 906                 pv->audio_passthru_slip += (start - sync->next_pts);
 907                 pv->video_pts_slip += (start - sync->next_pts);
 908                 return;
 909             }
 910             hb_log( "sync: adding %d ms of silence to audio %d"
 911                     "  start %"PRId64", next %"PRId64,
 912                     (int)((start - sync->next_pts) / 90),
 913                     i, start, sync->next_pts );
 914             InsertSilence( w, i, start - sync->next_pts );
 915             return;
 916         }
 917
 918         /*
 919          * When we get here we've taken care of all the dups and gaps in the
 920          * audio stream and are ready to inject the next input frame into
 921          * the output stream.
 922          */
 923         buf = hb_fifo_get( audio->priv.fifo_raw );
 924         OutputAudioFrame( job, audio, buf, sync, fifo, i );
 925     }
 926 }
 927
 928 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
 929 {
 930     hb_work_private_t * pv = w->private_data;
 931     hb_job_t        *job = pv->job;
 932     hb_sync_audio_t *sync = &pv->sync_audio[i];
 933     hb_buffer_t     *buf;
 934     hb_fifo_t       *fifo;
 935
 936     // to keep pass-thru and regular audio in sync we generate silence in
 937     // AC3 frame-sized units. If the silence duration isn't an integer multiple
 938     // of the AC3 frame duration we will truncate or round up depending on
 939     // which minimizes the timing error.
 940     const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
 941                           sync->audio->config.in.samplerate;
 942     int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
 943
 944     while ( --frame_count >= 0 )
 945     {
 946         if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
 947         {
 948             buf        = hb_buffer_init( sync->ac3_size );
 949             buf->start = sync->next_pts;
 950             buf->stop  = buf->start + frame_dur;
 951             memcpy( buf->data, sync->ac3_buf, buf->size );
 952             fifo = sync->audio->priv.fifo_out;
 953         }
 954         else
 955         {
 956             buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
 957                                      HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
 958                                          sync->audio->config.out.mixdown) );
 959             buf->start = sync->next_pts;
 960             buf->stop  = buf->start + frame_dur;
 961             memset( buf->data, 0, buf->size );
 962             fifo = sync->audio->priv.fifo_sync;
 963         }
 964         OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
 965     }
 966 }
 967
 968 static void UpdateState( hb_work_object_t * w )
 969 {
 970     hb_work_private_t * pv = w->private_data;
 971     hb_state_t state;
 972
 973     if( !pv->count_frames )
 974     {
 975         pv->st_first = hb_get_date();
 976         pv->job->st_pause_date = -1;
 977         pv->job->st_paused = 0;
 978     }
 979     pv->count_frames++;
 980
 981     if( hb_get_date() > pv->st_dates[3] + 1000 )
 982     {
 983         memmove( &pv->st_dates[0], &pv->st_dates[1],
 984                  3 * sizeof( uint64_t ) );
 985         memmove( &pv->st_counts[0], &pv->st_counts[1],
 986                  3 * sizeof( uint64_t ) );
 987         pv->st_dates[3]  = hb_get_date();
 988         pv->st_counts[3] = pv->count_frames;
 989     }
 990
 991 #define p state.param.working
 992     state.state = HB_STATE_WORKING;
 993     p.progress  = (float) pv->count_frames / (float) pv->count_frames_max;
 994     if( p.progress > 1.0 )
 995     {
 996         p.progress = 1.0;
 997     }
 998     p.rate_cur   = 1000.0 *
 999         (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
1000         (float) ( pv->st_dates[3] - pv->st_dates[0] );
1001     if( hb_get_date() > pv->st_first + 4000 )
1002     {
1003         int eta;
1004         p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
1005             (float) ( pv->st_dates[3] - pv->st_first - pv->job->st_paused);
1006         eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
1007             p.rate_avg;
1008         p.hours   = eta / 3600;
1009         p.minutes = ( eta % 3600 ) / 60;
1010         p.seconds = eta % 60;
1011     }
1012     else
1013     {
1014         p.rate_avg = 0.0;
1015         p.hours    = -1;
1016         p.minutes  = -1;
1017         p.seconds  = -1;
1018     }
1019 #undef p
1020
1021     hb_set_state( pv->job->h, &state );
1022 }