libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.m0k.org/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8
   9 #include "samplerate.h"
  10 #include "ffmpeg/avcodec.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_audio_t * audio;
  22     int64_t      count_frames;
  23
  24     /* Raw */
  25     SRC_STATE  * state;
  26     SRC_DATA     data;
  27
  28     /* AC-3 */
  29     int          ac3_size;
  30     uint8_t    * ac3_buf;
  31
  32 } hb_sync_audio_t;
  33
  34 struct hb_work_private_s
  35 {
  36     hb_job_t * job;
  37     int        done;
  38
  39     /* Video */
  40     hb_subtitle_t * subtitle;
  41     int64_t pts_offset;
  42     int64_t pts_offset_old;
  43     int64_t next_start;
  44     int64_t count_frames;
  45     int64_t count_frames_max;
  46     int64_t video_sequence;
  47     hb_buffer_t * cur; /* The next picture to process */
  48
  49     /* Audio */
  50     hb_sync_audio_t sync_audio[8];
  51
  52     /* Flags */
  53     int discontinuity;
  54
  55     /* Statistics */
  56     uint64_t st_counts[4];
  57     uint64_t st_dates[4];
  58     uint64_t st_first;
  59
  60     /* Throttle message flags */
  61     int   trashing_audio;
  62     int   inserting_silence;
  63     int   way_out_of_sync;
  64 };
  65
  66 /***********************************************************************
  67  * Local prototypes
  68  **********************************************************************/
  69 static void InitAudio( hb_work_object_t * w, int i );
  70 static int  SyncVideo( hb_work_object_t * w );
  71 static void SyncAudio( hb_work_object_t * w, int i );
  72 static int  NeedSilence( hb_work_object_t * w, hb_audio_t * );
  73 static void InsertSilence( hb_work_object_t * w, int i );
  74 static void UpdateState( hb_work_object_t * w );
  75
  76 /***********************************************************************
  77  * hb_work_sync_init
  78  ***********************************************************************
  79  * Initialize the work object
  80  **********************************************************************/
  81 int syncInit( hb_work_object_t * w, hb_job_t * job )
  82 {
  83     hb_title_t       * title = job->title;
  84     hb_chapter_t     * chapter;
  85     int                i;
  86     uint64_t           duration;
  87     hb_work_private_t * pv;
  88
  89     pv = calloc( 1, sizeof( hb_work_private_t ) );
  90     w->private_data = pv;
  91
  92     pv->job            = job;
  93     pv->pts_offset     = INT64_MIN;
  94     pv->pts_offset_old = INT64_MIN;
  95     pv->count_frames   = 0;
  96
  97     pv->discontinuity = 0;
  98
  99     pv->trashing_audio = 0;
 100     pv->inserting_silence = 0;
 101     pv->way_out_of_sync = 0;
 102
 103     /* Calculate how many video frames we are expecting */
 104     duration = 0;
 105     for( i = job->chapter_start; i <= job->chapter_end; i++ )
 106     {
 107         chapter   = hb_list_item( title->list_chapter, i - 1 );
 108         duration += chapter->duration;
 109     }
 110     duration += 90000;
 111         /* 1 second safety so we're sure we won't miss anything */
 112     pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
 113
 114     hb_log( "sync: expecting %lld video frames", pv->count_frames_max );
 115
 116     /* Initialize libsamplerate for every audio track we have */
 117     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 118     {
 119         InitAudio( w, i );
 120     }
 121
 122     /* Get subtitle info, if any */
 123     pv->subtitle = hb_list_item( title->list_subtitle, 0 );
 124
 125     pv->video_sequence = 0;
 126
 127     return 0;
 128 }
 129
 130 /***********************************************************************
 131  * Close
 132  ***********************************************************************
 133  *
 134  **********************************************************************/
 135 void syncClose( hb_work_object_t * w )
 136 {
 137     hb_work_private_t * pv = w->private_data;
 138     hb_job_t          * job   = pv->job;
 139     hb_title_t        * title = job->title;
 140
 141     int i;
 142
 143     if( pv->cur ) hb_buffer_close( &pv->cur );
 144
 145     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 146     {
 147         if( job->acodec & HB_ACODEC_AC3 ||
 148             job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 149         {
 150             free( pv->sync_audio[i].ac3_buf );
 151         }
 152         else
 153         {
 154             src_delete( pv->sync_audio[i].state );
 155         }
 156     }
 157
 158     free( pv );
 159     w->private_data = NULL;
 160 }
 161
 162 /***********************************************************************
 163  * Work
 164  ***********************************************************************
 165  * The root routine of this work abject
 166  *
 167  * The way this works is that we are syncing the audio to the PTS of
 168  * the last video that we processed. That's why we skip the audio sync
 169  * if we haven't got a valid PTS from the video yet.
 170  *
 171  **********************************************************************/
 172 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
 173               hb_buffer_t ** unused2 )
 174 {
 175     hb_work_private_t * pv = w->private_data;
 176     int i;
 177
 178     /* If we ever got a video frame, handle audio now */
 179     if( pv->pts_offset != INT64_MIN )
 180     {
 181         for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
 182         {
 183             SyncAudio( w, i );
 184         }
 185     }
 186
 187     /* Handle video */
 188     return SyncVideo( w );
 189 }
 190
 191 hb_work_object_t hb_sync =
 192 {
 193     WORK_SYNC,
 194     "Synchronization",
 195     syncInit,
 196     syncWork,
 197     syncClose
 198 };
 199
 200 static void InitAudio( hb_work_object_t * w, int i )
 201 {
 202     hb_work_private_t * pv = w->private_data;
 203     hb_job_t        * job   = pv->job;
 204     hb_title_t      * title = job->title;
 205     hb_sync_audio_t * sync;
 206
 207     sync        = &pv->sync_audio[i];
 208     sync->audio = hb_list_item( title->list_audio, i );
 209
 210     if( job->acodec & HB_ACODEC_AC3 ||
 211         job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 212     {
 213         /* Have a silent AC-3 frame ready in case we have to fill a
 214            gap */
 215         AVCodec        * codec;
 216         AVCodecContext * c;
 217         short          * zeros;
 218
 219         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 220         c     = avcodec_alloc_context();
 221
 222         c->bit_rate    = sync->audio->bitrate;
 223         c->sample_rate = sync->audio->rate;
 224         c->channels    = 2;
 225
 226         if( avcodec_open( c, codec ) < 0 )
 227         {
 228             hb_log( "sync: avcodec_open failed" );
 229             return;
 230         }
 231
 232         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 233                                  sizeof( short ) * c->channels, 1 );
 234         sync->ac3_size = sync->audio->bitrate * AC3_SAMPLES_PER_FRAME /
 235                              sync->audio->rate / 8;
 236         sync->ac3_buf  = malloc( sync->ac3_size );
 237
 238         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 239                                   zeros ) != sync->ac3_size )
 240         {
 241             hb_log( "sync: avcodec_encode_audio failed" );
 242         }
 243
 244         free( zeros );
 245         avcodec_close( c );
 246         av_free( c );
 247     }
 248     else
 249     {
 250         /* Initialize libsamplerate */
 251         int error;
 252         sync->state             = src_new( SRC_LINEAR, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->amixdown), &error );
 253         sync->data.end_of_input = 0;
 254     }
 255 }
 256
 257
 258
 259 #define PTS_DISCONTINUITY_TOLERANCE 90000
 260
 261 /***********************************************************************
 262  * SyncVideo
 263  ***********************************************************************
 264  *
 265  **********************************************************************/
 266 static int SyncVideo( hb_work_object_t * w )
 267 {
 268     hb_work_private_t * pv = w->private_data;
 269     hb_buffer_t * cur, * next, * sub = NULL;
 270     hb_job_t * job = pv->job;
 271     int64_t pts_expected;
 272     int chap_break;
 273
 274     if( pv->done )
 275     {
 276         return HB_WORK_DONE;
 277     }
 278
 279     if( hb_thread_has_exited( job->reader ) &&
 280         !hb_fifo_size( job->fifo_mpeg2 ) &&
 281         !hb_fifo_size( job->fifo_raw ) )
 282     {
 283         /* All video data has been processed already, we won't get
 284            more */
 285         hb_log( "sync: got %lld frames, %lld expected",
 286                 pv->count_frames, pv->count_frames_max );
 287         pv->done = 1;
 288
 289         hb_buffer_t * buf_tmp;
 290
 291        // Drop an empty buffer into our output to ensure that things
 292        // get flushed all the way out.
 293         buf_tmp = hb_buffer_init(0); // Empty end buffer
 294         hb_fifo_push( job->fifo_sync, buf_tmp );
 295
 296         return HB_WORK_DONE;
 297     }
 298
 299     if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
 300     {
 301         /* We haven't even got a frame yet */
 302         return HB_WORK_OK;
 303     }
 304     cur = pv->cur;
 305
 306     /* At this point we have a frame to process. Let's check
 307         1) if we will be able to push into the fifo ahead
 308         2) if the next frame is there already, since we need it to
 309            know whether we'll have to repeat the current frame or not */
 310     while( !hb_fifo_is_full( job->fifo_sync ) &&
 311            ( next = hb_fifo_see( job->fifo_raw ) ) )
 312     {
 313         hb_buffer_t * buf_tmp;
 314
 315         if( pv->pts_offset == INT64_MIN )
 316         {
 317             /* This is our first frame */
 318             hb_log( "sync: first pts is %lld", cur->start );
 319             pv->pts_offset = cur->start;
 320         }
 321
 322         /*
 323          * Track the video sequence number localy so that we can sync the audio
 324          * to it using the sequence number as well as the PTS.
 325          */
 326         pv->video_sequence = cur->sequence;
 327
 328         /* Check for PTS jumps over 0.5 second */
 329         if( next->start < cur->start - PTS_DISCONTINUITY_TOLERANCE ||
 330             next->start > cur->start + PTS_DISCONTINUITY_TOLERANCE )
 331         {
 332             hb_log( "Sync: Video PTS discontinuity %s (current buffer start=%lld, next buffer start=%lld)",
 333                     pv->discontinuity ? "second" : "first", cur->start, next->start );
 334
 335             /*
 336              * Do we need to trash the subtitle, is it from the next->start period
 337              * or is it from our old position. If the latter then trash it.
 338              */
 339             if( pv->subtitle )
 340             {
 341                 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
 342                 {
 343                     if( ( sub->start > ( cur->start - PTS_DISCONTINUITY_TOLERANCE ) ) &&
 344                         ( sub->start < ( cur->start + PTS_DISCONTINUITY_TOLERANCE ) ) )
 345                     {
 346                         /*
 347                          * The subtitle is from our current time region which we are
 348                          * jumping from. So trash it as we are about to jump backwards
 349                          * or forwards and don't want it blocking the subtitle fifo.
 350                          */
 351                         hb_log("Trashing subtitle 0x%x due to PTS discontinuity", sub);
 352                         sub = hb_fifo_get( pv->subtitle->fifo_raw );
 353                         hb_buffer_close( &sub );
 354                     } else {
 355                         break;
 356                     }
 357                 }
 358             }
 359
 360             /* Trash current picture */
 361             /* Also, make sure we don't trash a chapter break */
 362             chap_break = cur->new_chap;
 363             hb_buffer_close( &cur );
 364             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 365             cur->new_chap |= chap_break; // Don't stomp existing chapter breaks
 366
 367             /* Calculate new offset */
 368             pv->pts_offset_old = pv->pts_offset;
 369             if ( job->vfr )
 370             {
 371                 pv->pts_offset = cur->start - pv->next_start;
 372             } else {
 373                 pv->pts_offset = cur->start -
 374                     pv->count_frames * pv->job->vrate_base / 300;
 375             }
 376
 377             if( !pv->discontinuity )
 378             {
 379                 pv->discontinuity = 1;
 380             }
 381
 382             pv->video_sequence = cur->sequence;
 383             continue;
 384         }
 385
 386         /* Look for a subtitle for this frame */
 387         if( pv->subtitle )
 388         {
 389             hb_buffer_t * sub2;
 390             while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
 391             {
 392                 /* If two subtitles overlap, make the first one stop
 393                    when the second one starts */
 394                 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
 395                 if( sub2 && sub->stop > sub2->start )
 396                     sub->stop = sub2->start;
 397
 398                 // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 399                 //       sub, cur->sequence, sub->sequence);
 400
 401                 if( sub->sequence > cur->sequence )
 402                 {
 403                     /*
 404                      * The video is behind where we are, so wait until
 405                      * it catches up to the same reader point on the
 406                      * DVD. Then our PTS should be in the same region
 407                      * as the video.
 408                      */
 409                     sub = NULL;
 410                     break;
 411                 }
 412
 413                 if( sub->stop > cur->start ) {
 414                     /*
 415                      * The stop time is in the future, so fall through
 416                      * and we'll deal with it in the next block of
 417                      * code.
 418                      */
 419                     break;
 420                 }
 421                 else
 422                 {
 423                     /*
 424                      * The stop time is in the past. But is it due to
 425                      * it having been played already, or has the PTS
 426                      * been reset to 0?
 427                      */
 428                     if( ( cur->start - sub->stop ) > PTS_DISCONTINUITY_TOLERANCE ) {
 429                         /*
 430                          * There is a lot of time between our current
 431                          * video and where this subtitle is ending,
 432                          * assume that we are about to reset the PTS
 433                          * and do not throw away this subtitle.
 434                          */
 435                         break;
 436                     }
 437                 }
 438
 439                 /*
 440                  * The subtitle is older than this picture, trash it
 441                  */
 442                 sub = hb_fifo_get( pv->subtitle->fifo_raw );
 443                 hb_buffer_close( &sub );
 444             }
 445
 446             /*
 447              * There is a valid subtitle, is it time to display it?
 448              */
 449             if( sub )
 450             {
 451                 if( sub->stop > sub->start)
 452                 {
 453                     /*
 454                      * Normal subtitle which ends after it starts, check to
 455                      * see that the current video is between the start and end.
 456                      */
 457                     if( cur->start > sub->start &&
 458                         cur->start < sub->stop )
 459                     {
 460                         /*
 461                          * We should be playing this, so leave the
 462                          * subtitle in place.
 463                          *
 464                          * fall through to display
 465                          */
 466                     }
 467                     else
 468                     {
 469                         /*
 470                          * Defer until the play point is within the subtitle
 471                          */
 472                         sub = NULL;
 473                     }
 474                 }
 475                 else
 476                 {
 477                     /*
 478                      * The end of the subtitle is less than the start, this is a
 479                      * sign of a PTS discontinuity.
 480                      */
 481                     if( sub->start > cur->start )
 482                     {
 483                         /*
 484                          * we haven't reached the start time yet, or
 485                          * we have jumped backwards after having
 486                          * already started this subtitle.
 487                          */
 488                         if( cur->start < sub->stop )
 489                         {
 490                             /*
 491                              * We have jumped backwards and so should
 492                              * continue displaying this subtitle.
 493                              *
 494                              * fall through to display.
 495                              */
 496                         }
 497                         else
 498                         {
 499                             /*
 500                              * Defer until the play point is within the subtitle
 501                              */
 502                             sub = NULL;
 503                         }
 504                     } else {
 505                         /*
 506                          * Play this subtitle as the start is greater than our
 507                          * video point.
 508                          *
 509                          * fall through to display/
 510                          */
 511                     }
 512                 }
 513             }
 514         }
 515
 516         if ( job->vfr )
 517         {
 518             /*
 519              * adjust the pts of the current frame so that it's contiguous
 520              * with the previous frame. pts_offset tracks the time difference
 521              * between the pts values in the input content (which start at some
 522              * random time) and our timestamps (which start at zero). We don't
 523              * make any adjustments to the source timestamps other than removing
 524              * the clock offsets (which also removes pts discontinuities).
 525              * This means we automatically encode at the source's frame rate.
 526              * MP2 uses an implicit duration (frames end when the next frame
 527              * starts) but more advanced containers like MP4 use an explicit
 528              * duration. Since we're looking ahead one frame we set the
 529              * explicit stop time from the start time of the next frame.
 530              */
 531             buf_tmp = cur;
 532             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 533             buf_tmp->start = pv->next_start;
 534             pv->next_start = next->start - pv->pts_offset;
 535             buf_tmp->stop = pv->next_start;
 536         }
 537         else
 538         {
 539             /* The PTS of the frame we are expecting now */
 540             pts_expected = pv->pts_offset +
 541                 pv->count_frames * pv->job->vrate_base / 300;
 542
 543             //hb_log("Video expecting PTS %lld, current frame: %lld, next frame: %lld, cf: %lld",
 544             //       pts_expected, cur->start, next->start, pv->count_frames * pv->job->vrate_base / 300 );
 545
 546             if( cur->start < pts_expected - pv->job->vrate_base / 300 / 2 &&
 547                 next->start < pts_expected + pv->job->vrate_base / 300 / 2 )
 548             {
 549                 /* The current frame is too old but the next one matches,
 550                    let's trash */
 551                 /* Also, make sure we don't trash a chapter break */
 552                 chap_break = cur->new_chap;
 553                 hb_buffer_close( &cur );
 554                 pv->cur = cur = hb_fifo_get( job->fifo_raw );
 555                 cur->new_chap |= chap_break; // Make sure we don't stomp the existing one.
 556
 557                 continue;
 558             }
 559
 560             if( next->start > pts_expected + 3 * pv->job->vrate_base / 300 / 2 )
 561             {
 562                 /* We'll need the current frame more than one time. Make a
 563                    copy of it and keep it */
 564                 buf_tmp = hb_buffer_init( cur->size );
 565                 memcpy( buf_tmp->data, cur->data, cur->size );
 566                 buf_tmp->sequence = cur->sequence;
 567             }
 568             else
 569             {
 570                 /* The frame has the expected date and won't have to be
 571                    duplicated, just put it through */
 572                 buf_tmp = cur;
 573                 pv->cur = cur = hb_fifo_get( job->fifo_raw );
 574             }
 575
 576             /* Replace those MPEG-2 dates with our dates */
 577             buf_tmp->start = (uint64_t) pv->count_frames *
 578                 pv->job->vrate_base / 300;
 579             buf_tmp->stop  = (uint64_t) ( pv->count_frames + 1 ) *
 580                 pv->job->vrate_base / 300;
 581         }
 582
 583         /* If we have a subtitle for this picture, copy it */
 584         /* FIXME: we should avoid this memcpy */
 585         if( sub )
 586         {
 587             buf_tmp->sub         = hb_buffer_init( sub->size );
 588             buf_tmp->sub->x      = sub->x;
 589             buf_tmp->sub->y      = sub->y;
 590             buf_tmp->sub->width  = sub->width;
 591             buf_tmp->sub->height = sub->height;
 592             memcpy( buf_tmp->sub->data, sub->data, sub->size );
 593         }
 594
 595         /* Push the frame to the renderer */
 596         hb_fifo_push( job->fifo_sync, buf_tmp );
 597
 598         /* Update UI */
 599         UpdateState( w );
 600
 601         /* Make sure we won't get more frames then expected */
 602         if( pv->count_frames >= pv->count_frames_max * 2)
 603         {
 604             hb_log( "sync: got too many frames (%lld), exiting early", pv->count_frames );
 605             pv->done = 1;
 606
 607            // Drop an empty buffer into our output to ensure that things
 608            // get flushed all the way out.
 609            buf_tmp = hb_buffer_init(0); // Empty end buffer
 610            hb_fifo_push( job->fifo_sync, buf_tmp );
 611
 612             break;
 613         }
 614     }
 615
 616     return HB_WORK_OK;
 617 }
 618
 619 /***********************************************************************
 620  * SyncAudio
 621  ***********************************************************************
 622  *
 623  **********************************************************************/
 624 static void SyncAudio( hb_work_object_t * w, int i )
 625 {
 626     hb_work_private_t * pv = w->private_data;
 627     hb_job_t        * job;
 628     hb_audio_t      * audio;
 629     hb_buffer_t     * buf;
 630     hb_sync_audio_t * sync;
 631
 632     hb_fifo_t       * fifo;
 633     int               rate;
 634
 635     int64_t           pts_expected;
 636     int64_t           start;
 637
 638     job    = pv->job;
 639     sync   = &pv->sync_audio[i];
 640     audio  = sync->audio;
 641
 642     if( job->acodec & HB_ACODEC_AC3 ||
 643         job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 644     {
 645         fifo = audio->fifo_out;
 646         rate = audio->rate;
 647     }
 648     else
 649     {
 650         fifo = audio->fifo_sync;
 651         rate = job->arate;
 652     }
 653
 654     while( !hb_fifo_is_full( fifo ) &&
 655            ( buf = hb_fifo_see( audio->fifo_raw ) ) )
 656     {
 657         /* The PTS of the samples we are expecting now */
 658         pts_expected = pv->pts_offset + sync->count_frames * 90000 / rate;
 659
 660         // hb_log("Video Sequence: %lld, Audio Sequence: %lld", pv->video_sequence, buf->sequence);
 661
 662         /*
 663          * Using the same logic as the Video have we crossed a VOB
 664          * boundary as detected by the expected PTS and the PTS of our
 665          * audio being out by more than the tolerance value.
 666          */
 667         if( buf->start > pts_expected + PTS_DISCONTINUITY_TOLERANCE ||
 668             buf->start < pts_expected - PTS_DISCONTINUITY_TOLERANCE )
 669         {
 670             /* There has been a PTS discontinuity, and this frame might
 671                be from before the discontinuity*/
 672
 673             if( pv->discontinuity )
 674             {
 675                 /*
 676                  * There is an outstanding discontinuity, so use the offset from
 677                  * that discontinuity.
 678                  */
 679                 pts_expected = pv->pts_offset_old + sync->count_frames *
 680                     90000 / rate;
 681             }
 682             else
 683             {
 684                 /*
 685                  * No outstanding discontinuity, so the audio must be leading the
 686                  * video (or the PTS values are really stuffed). So lets mark this
 687                  * as a discontinuity ourselves for the audio to use until
 688                  * the video also crosses the discontinuity.
 689                  *
 690                  * pts_offset is used when we are in the same time space as the video
 691                  * pts_offset_old when in a discontinuity.
 692                  *
 693                  * Therefore set the pts_offset_old given the new pts_offset for this
 694                  * current buffer.
 695                  */
 696                 pv->discontinuity = 1;
 697                 pv->pts_offset_old = buf->start - sync->count_frames *
 698                     90000 / rate;
 699                 pts_expected = pv->pts_offset_old + sync->count_frames *
 700                     90000 / rate;
 701
 702                 hb_log("Sync: Audio discontinuity (sequence: vid %lld aud %lld) (pts %lld < %lld < %lld)",
 703                        pv->video_sequence, buf->sequence,
 704                        pts_expected - PTS_DISCONTINUITY_TOLERANCE, buf->start,
 705                        pts_expected + PTS_DISCONTINUITY_TOLERANCE );
 706             }
 707
 708             /*
 709              * Is the audio from a valid period given the previous
 710              * Video PTS. I.e. has there just been a video PTS
 711              * discontinuity and this audio belongs to the vdeo from
 712              * before?
 713              */
 714             if( buf->start > pts_expected + PTS_DISCONTINUITY_TOLERANCE ||
 715                 buf->start < pts_expected - PTS_DISCONTINUITY_TOLERANCE )
 716             {
 717                 /*
 718                  * It's outside of our tolerance for where the video
 719                  * is now, and it's outside of the tolerance for
 720                  * where we have been in the case of a VOB change.
 721                  * Try and reconverge regardless. so continue on to
 722                  * our convergence code below which will kick in as
 723                  * it will be more than 100ms out.
 724                  *
 725                  * Note that trashing the Audio could make things
 726                  * worse if the Audio is in front because we will end
 727                  * up diverging even more. We need to hold on to the
 728                  * audio until the video catches up.
 729                  */
 730                 if( !pv->way_out_of_sync )
 731                 {
 732                     hb_log("Sync: Audio is way out of sync, attempt to reconverge from current video PTS");
 733                     pv->way_out_of_sync = 1;
 734                 }
 735
 736                 /*
 737                  * It wasn't from the old place, so we must be from
 738                  * the new, but just too far out. So attempt to
 739                  * reconverge by resetting the point we want to be to
 740                  * where we are currently wanting to be.
 741                  */
 742                 pts_expected = pv->pts_offset + sync->count_frames * 90000 / rate;
 743                 start = pts_expected - pv->pts_offset;
 744             } else {
 745                  /* Use the older offset */
 746                 start = pts_expected - pv->pts_offset_old;
 747             }
 748         }
 749         else
 750         {
 751             start = pts_expected - pv->pts_offset;
 752
 753             if( pv->discontinuity )
 754             {
 755                 /*
 756                  * The Audio is tracking the Video again using the normal pts_offset, so the
 757                  * discontinuity is over.
 758                  */
 759                 hb_log( "Sync: Audio joined Video after discontinuity at PTS %lld", buf->start );
 760                 pv->discontinuity = 0;
 761             }
 762         }
 763
 764         /* Tolerance: 100 ms */
 765         if( buf->start < pts_expected - 9000 )
 766         {
 767             if( !pv->trashing_audio )
 768             {
 769                 /* Audio is behind the Video, trash it, can't use it now. */
 770                 hb_log( "Sync: Audio PTS (%lld) < Video PTS (%lld) by greater than 100ms, trashing audio to reconverge",
 771                         buf->start, pts_expected);
 772                 pv->trashing_audio = 1;
 773             }
 774             buf = hb_fifo_get( audio->fifo_raw );
 775             hb_buffer_close( &buf );
 776             continue;
 777         }
 778         else if( buf->start > pts_expected + 9000 )
 779         {
 780             /* Audio is ahead of the Video, insert silence until we catch up*/
 781             if( !pv->inserting_silence )
 782             {
 783                 hb_log("Sync: Audio PTS (%lld) >  Video PTS (%lld) by greater than 100ms insert silence until reconverged", buf->start, pts_expected);
 784                 pv->inserting_silence = 1;
 785             }
 786             InsertSilence( w, i );
 787             continue;
 788         }
 789         else
 790         {
 791             if( pv->trashing_audio || pv->inserting_silence )
 792             {
 793                 hb_log( "Sync: Audio back in Sync at PTS %lld", buf->start );
 794                 pv->trashing_audio = 0;
 795                 pv->inserting_silence = 0;
 796             }
 797             if( pv->way_out_of_sync )
 798             {
 799                 hb_log( "Sync: Audio no longer way out of sync at PTS %lld",
 800                         buf->start );
 801                 pv->way_out_of_sync = 0;
 802             }
 803         }
 804
 805         if( job->acodec & HB_ACODEC_AC3 ||
 806             job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 807         {
 808             buf        = hb_fifo_get( audio->fifo_raw );
 809             buf->start = start;
 810             buf->stop  = start + 90000 * AC3_SAMPLES_PER_FRAME / rate;
 811
 812             sync->count_frames += AC3_SAMPLES_PER_FRAME;
 813         }
 814         else
 815         {
 816             hb_buffer_t * buf_raw = hb_fifo_get( audio->fifo_raw );
 817
 818             int count_in, count_out;
 819
 820             count_in  = buf_raw->size / HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) / sizeof( float );
 821             count_out = ( buf_raw->stop - buf_raw->start ) * job->arate / 90000;
 822             if( buf->start < pts_expected - 1500 )
 823                 count_out--;
 824             else if( buf->start > pts_expected + 1500 )
 825                 count_out++;
 826
 827             sync->data.data_in      = (float *) buf_raw->data;
 828             sync->data.input_frames = count_in;
 829             sync->data.output_frames = count_out;
 830
 831             sync->data.src_ratio = (double) sync->data.output_frames /
 832                                    (double) sync->data.input_frames;
 833
 834             buf = hb_buffer_init( sync->data.output_frames * HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) *
 835                                   sizeof( float ) );
 836             sync->data.data_out = (float *) buf->data;
 837             if( src_process( sync->state, &sync->data ) )
 838             {
 839                 /* XXX If this happens, we're screwed */
 840                 hb_log( "sync: src_process failed" );
 841             }
 842             hb_buffer_close( &buf_raw );
 843
 844             buf->size = sync->data.output_frames_gen * HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) * sizeof( float );
 845
 846             /* Set dates for resampled data */
 847             buf->start = start;
 848             buf->stop  = start + sync->data.output_frames_gen *
 849                             90000 / job->arate;
 850
 851             sync->count_frames += sync->data.output_frames_gen;
 852         }
 853
 854         buf->frametype = HB_FRAME_AUDIO;
 855         hb_fifo_push( fifo, buf );
 856     }
 857
 858     if( hb_fifo_is_full( fifo ) &&
 859         pv->way_out_of_sync )
 860     {
 861         /*
 862          * Trash the top audio packet to avoid dead lock as we reconverge.
 863          */
 864         if ( (buf = hb_fifo_get( audio->fifo_raw ) ) != NULL)
 865             hb_buffer_close( &buf );
 866     }
 867
 868     if( NeedSilence( w, audio ) )
 869     {
 870         InsertSilence( w, i );
 871     }
 872 }
 873
 874 static int NeedSilence( hb_work_object_t * w, hb_audio_t * audio )
 875 {
 876     hb_work_private_t * pv = w->private_data;
 877     hb_job_t * job = pv->job;
 878
 879     if( hb_fifo_size( audio->fifo_in ) ||
 880         hb_fifo_size( audio->fifo_raw ) ||
 881         hb_fifo_size( audio->fifo_sync ) ||
 882         hb_fifo_size( audio->fifo_out ) )
 883     {
 884         /* We have some audio, we are fine */
 885         return 0;
 886     }
 887
 888     /* No audio left in fifos */
 889
 890     if( hb_thread_has_exited( job->reader ) )
 891     {
 892         /* We might miss some audio to complete encoding and muxing
 893            the video track */
 894         hb_log("Reader has exited early, inserting silence.");
 895         return 1;
 896     }
 897
 898     if( hb_fifo_is_full( job->fifo_mpeg2 ) &&
 899         hb_fifo_is_full( job->fifo_raw ) &&
 900         hb_fifo_is_full( job->fifo_sync ) &&
 901         hb_fifo_is_full( job->fifo_render ) &&
 902         hb_fifo_is_full( job->fifo_mpeg4 ) )
 903     {
 904         /* Too much video and no audio, oh-oh */
 905         hb_log("Still got some video - and nothing in the audio fifo, insert silence");
 906         return 1;
 907     }
 908
 909     return 0;
 910 }
 911
 912 static void InsertSilence( hb_work_object_t * w, int i )
 913 {
 914     hb_work_private_t * pv = w->private_data;
 915     hb_job_t        * job;
 916     hb_sync_audio_t * sync;
 917     hb_buffer_t     * buf;
 918
 919     job    = pv->job;
 920     sync   = &pv->sync_audio[i];
 921
 922     if( job->acodec & HB_ACODEC_AC3 ||
 923         job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 924     {
 925         buf        = hb_buffer_init( sync->ac3_size );
 926         buf->start = sync->count_frames * 90000 / sync->audio->rate;
 927         buf->stop  = buf->start + 90000 * AC3_SAMPLES_PER_FRAME /
 928                      sync->audio->rate;
 929         memcpy( buf->data, sync->ac3_buf, buf->size );
 930
 931         hb_log( "sync: adding a silent AC-3 frame for track %x",
 932                 sync->audio->id );
 933         hb_fifo_push( sync->audio->fifo_out, buf );
 934
 935         sync->count_frames += AC3_SAMPLES_PER_FRAME;
 936
 937     }
 938     else
 939     {
 940         buf        = hb_buffer_init( HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->amixdown) * job->arate / 20 *
 941                                      sizeof( float ) );
 942         buf->start = sync->count_frames * 90000 / job->arate;
 943         buf->stop  = buf->start + 90000 / 20;
 944         memset( buf->data, 0, buf->size );
 945
 946         hb_fifo_push( sync->audio->fifo_sync, buf );
 947
 948         sync->count_frames += job->arate / 20;
 949     }
 950 }
 951
 952 static void UpdateState( hb_work_object_t * w )
 953 {
 954     hb_work_private_t * pv = w->private_data;
 955     hb_state_t state;
 956
 957     if( !pv->count_frames )
 958     {
 959         pv->st_first = hb_get_date();
 960     }
 961     pv->count_frames++;
 962
 963     if( hb_get_date() > pv->st_dates[3] + 1000 )
 964     {
 965         memmove( &pv->st_dates[0], &pv->st_dates[1],
 966                  3 * sizeof( uint64_t ) );
 967         memmove( &pv->st_counts[0], &pv->st_counts[1],
 968                  3 * sizeof( uint64_t ) );
 969         pv->st_dates[3]  = hb_get_date();
 970         pv->st_counts[3] = pv->count_frames;
 971     }
 972
 973 #define p state.param.working
 974     state.state = HB_STATE_WORKING;
 975     p.progress  = (float) pv->count_frames / (float) pv->count_frames_max;
 976     if( p.progress > 1.0 )
 977     {
 978         p.progress = 1.0;
 979     }
 980     p.rate_cur   = 1000.0 *
 981         (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
 982         (float) ( pv->st_dates[3] - pv->st_dates[0] );
 983     if( hb_get_date() > pv->st_first + 4000 )
 984     {
 985         int eta;
 986         p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
 987             (float) ( pv->st_dates[3] - pv->st_first );
 988         eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
 989             p.rate_avg;
 990         p.hours   = eta / 3600;
 991         p.minutes = ( eta % 3600 ) / 60;
 992         p.seconds = eta % 60;
 993     }
 994     else
 995     {
 996         p.rate_avg = 0.0;
 997         p.hours    = -1;
 998         p.minutes  = -1;
 999         p.seconds  = -1;
1000     }
1001 #undef p
1002
1003     hb_set_state( pv->job->h, &state );
1004 }