libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.m0k.org/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8
   9 #include "samplerate.h"
  10 #include "ffmpeg/avcodec.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_audio_t * audio;
  22     int64_t      count_frames;
  23
  24     /* Raw */
  25     SRC_STATE  * state;
  26     SRC_DATA     data;
  27
  28     /* AC-3 */
  29     int          ac3_size;
  30     uint8_t    * ac3_buf;
  31
  32 } hb_sync_audio_t;
  33
  34 struct hb_work_private_s
  35 {
  36     hb_job_t * job;
  37     int        done;
  38
  39     /* Video */
  40     hb_subtitle_t * subtitle;
  41     int64_t pts_offset;
  42     int64_t pts_offset_old;
  43     int64_t count_frames;
  44     int64_t count_frames_max;
  45     int64_t video_sequence;
  46     hb_buffer_t * cur; /* The next picture to process */
  47
  48     /* Audio */
  49     hb_sync_audio_t sync_audio[8];
  50
  51     /* Flags */
  52     int discontinuity;
  53
  54     /* Statistics */
  55     uint64_t st_counts[4];
  56     uint64_t st_dates[4];
  57     uint64_t st_first;
  58
  59     /* Throttle message flags */
  60     int   trashing_audio;
  61     int   inserting_silence;
  62     int   way_out_of_sync;
  63 };
  64
  65 /***********************************************************************
  66  * Local prototypes
  67  **********************************************************************/
  68 static void InitAudio( hb_work_object_t * w, int i );
  69 static int  SyncVideo( hb_work_object_t * w );
  70 static void SyncAudio( hb_work_object_t * w, int i );
  71 static int  NeedSilence( hb_work_object_t * w, hb_audio_t * );
  72 static void InsertSilence( hb_work_object_t * w, int i );
  73 static void UpdateState( hb_work_object_t * w );
  74
  75 /***********************************************************************
  76  * hb_work_sync_init
  77  ***********************************************************************
  78  * Initialize the work object
  79  **********************************************************************/
  80 int syncInit( hb_work_object_t * w, hb_job_t * job )
  81 {
  82     hb_title_t       * title = job->title;
  83     hb_chapter_t     * chapter;
  84     int                i;
  85     uint64_t           duration;
  86     hb_work_private_t * pv;
  87
  88     pv = calloc( 1, sizeof( hb_work_private_t ) );
  89     w->private_data = pv;
  90
  91     pv->job            = job;
  92     pv->pts_offset     = INT64_MIN;
  93     pv->pts_offset_old = INT64_MIN;
  94     pv->count_frames   = 0;
  95
  96     pv->discontinuity = 0;
  97
  98     pv->trashing_audio = 0;
  99     pv->inserting_silence = 0;
 100     pv->way_out_of_sync = 0;
 101
 102     /* Calculate how many video frames we are expecting */
 103     duration = 0;
 104     for( i = job->chapter_start; i <= job->chapter_end; i++ )
 105     {
 106         chapter   = hb_list_item( title->list_chapter, i - 1 );
 107         duration += chapter->duration;
 108     }
 109     duration += 90000;
 110         /* 1 second safety so we're sure we won't miss anything */
 111     pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
 112
 113     hb_log( "sync: expecting %lld video frames", pv->count_frames_max );
 114
 115     /* Initialize libsamplerate for every audio track we have */
 116     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 117     {
 118         InitAudio( w, i );
 119     }
 120
 121     /* Get subtitle info, if any */
 122     pv->subtitle = hb_list_item( title->list_subtitle, 0 );
 123
 124     pv->video_sequence = 0;
 125
 126     return 0;
 127 }
 128
 129 /***********************************************************************
 130  * Close
 131  ***********************************************************************
 132  *
 133  **********************************************************************/
 134 void syncClose( hb_work_object_t * w )
 135 {
 136     hb_work_private_t * pv = w->private_data;
 137     hb_job_t          * job   = pv->job;
 138     hb_title_t        * title = job->title;
 139
 140     int i;
 141
 142     if( pv->cur ) hb_buffer_close( &pv->cur );
 143
 144     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 145     {
 146         if( job->acodec & HB_ACODEC_AC3 )
 147         {
 148             free( pv->sync_audio[i].ac3_buf );
 149         }
 150         else
 151         {
 152             src_delete( pv->sync_audio[i].state );
 153         }
 154     }
 155
 156     free( pv );
 157     w->private_data = NULL;
 158 }
 159
 160 /***********************************************************************
 161  * Work
 162  ***********************************************************************
 163  * The root routine of this work abject
 164  *
 165  * The way this works is that we are syncing the audio to the PTS of
 166  * the last video that we processed. That's why we skip the audio sync
 167  * if we haven't got a valid PTS from the video yet.
 168  *
 169  **********************************************************************/
 170 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
 171               hb_buffer_t ** unused2 )
 172 {
 173     hb_work_private_t * pv = w->private_data;
 174     int i;
 175
 176     /* If we ever got a video frame, handle audio now */
 177     if( pv->pts_offset != INT64_MIN )
 178     {
 179         for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
 180         {
 181             SyncAudio( w, i );
 182         }
 183     }
 184
 185     /* Handle video */
 186     return SyncVideo( w );
 187 }
 188
 189 hb_work_object_t hb_sync =
 190 {
 191     WORK_SYNC,
 192     "Synchronization",
 193     syncInit,
 194     syncWork,
 195     syncClose
 196 };
 197
 198 static void InitAudio( hb_work_object_t * w, int i )
 199 {
 200     hb_work_private_t * pv = w->private_data;
 201     hb_job_t        * job   = pv->job;
 202     hb_title_t      * title = job->title;
 203     hb_sync_audio_t * sync;
 204
 205     sync        = &pv->sync_audio[i];
 206     sync->audio = hb_list_item( title->list_audio, i );
 207
 208     if( job->acodec & HB_ACODEC_AC3 )
 209     {
 210         /* Have a silent AC-3 frame ready in case we have to fill a
 211            gap */
 212         AVCodec        * codec;
 213         AVCodecContext * c;
 214         short          * zeros;
 215
 216         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 217         c     = avcodec_alloc_context();
 218
 219         c->bit_rate    = sync->audio->bitrate;
 220         c->sample_rate = sync->audio->rate;
 221         c->channels    = 2;
 222
 223         if( avcodec_open( c, codec ) < 0 )
 224         {
 225             hb_log( "sync: avcodec_open failed" );
 226             return;
 227         }
 228
 229         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 230                                  sizeof( short ) * c->channels, 1 );
 231         sync->ac3_size = sync->audio->bitrate * AC3_SAMPLES_PER_FRAME /
 232                              sync->audio->rate / 8;
 233         sync->ac3_buf  = malloc( sync->ac3_size );
 234
 235         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 236                                   zeros ) != sync->ac3_size )
 237         {
 238             hb_log( "sync: avcodec_encode_audio failed" );
 239         }
 240
 241         free( zeros );
 242         avcodec_close( c );
 243         av_free( c );
 244     }
 245     else
 246     {
 247         /* Initialize libsamplerate */
 248         int error;
 249         sync->state             = src_new( SRC_LINEAR, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->amixdown), &error );
 250         sync->data.end_of_input = 0;
 251     }
 252 }
 253
 254
 255
 256 #define PTS_DISCONTINUITY_TOLERANCE 90000
 257
 258 /***********************************************************************
 259  * SyncVideo
 260  ***********************************************************************
 261  *
 262  **********************************************************************/
 263 static int SyncVideo( hb_work_object_t * w )
 264 {
 265     hb_work_private_t * pv = w->private_data;
 266     hb_buffer_t * cur, * next, * sub = NULL;
 267     hb_job_t * job = pv->job;
 268     int64_t pts_expected;
 269     int chap_break;
 270
 271     if( pv->done )
 272     {
 273         return HB_WORK_DONE;
 274     }
 275
 276     if( hb_thread_has_exited( job->reader ) &&
 277         !hb_fifo_size( job->fifo_mpeg2 ) &&
 278         !hb_fifo_size( job->fifo_raw ) )
 279     {
 280         /* All video data has been processed already, we won't get
 281            more */
 282         hb_log( "sync: got %lld frames, %lld expected",
 283                 pv->count_frames, pv->count_frames_max );
 284         pv->done = 1;
 285
 286         hb_buffer_t * buf_tmp;
 287
 288        // Drop an empty buffer into our output to ensure that things
 289        // get flushed all the way out.
 290         buf_tmp = hb_buffer_init(0); // Empty end buffer
 291         hb_fifo_push( job->fifo_sync, buf_tmp );
 292
 293         return HB_WORK_DONE;
 294     }
 295
 296     if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
 297     {
 298         /* We haven't even got a frame yet */
 299         return HB_WORK_OK;
 300     }
 301     cur = pv->cur;
 302
 303     /* At this point we have a frame to process. Let's check
 304         1) if we will be able to push into the fifo ahead
 305         2) if the next frame is there already, since we need it to
 306            know whether we'll have to repeat the current frame or not */
 307     while( !hb_fifo_is_full( job->fifo_sync ) &&
 308            ( next = hb_fifo_see( job->fifo_raw ) ) )
 309     {
 310         hb_buffer_t * buf_tmp;
 311
 312         if( pv->pts_offset == INT64_MIN )
 313         {
 314             /* This is our first frame */
 315             hb_log( "sync: first pts is %lld", cur->start );
 316             pv->pts_offset = cur->start;
 317         }
 318
 319         /*
 320          * Track the video sequence number localy so that we can sync the audio
 321          * to it using the sequence number as well as the PTS.
 322          */
 323         pv->video_sequence = cur->sequence;
 324
 325         /* Check for PTS jumps over 0.5 second */
 326         if( next->start < cur->start - PTS_DISCONTINUITY_TOLERANCE ||
 327             next->start > cur->start + PTS_DISCONTINUITY_TOLERANCE )
 328         {
 329             hb_log( "Sync: Video PTS discontinuity %s (current buffer start=%lld, next buffer start=%lld)",
 330                     pv->discontinuity ? "second" : "first", cur->start, next->start );
 331
 332             /*
 333              * Do we need to trash the subtitle, is it from the next->start period
 334              * or is it from our old position. If the latter then trash it.
 335              */
 336             if( pv->subtitle )
 337             {
 338                 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
 339                 {
 340                     if( ( sub->start > ( cur->start - PTS_DISCONTINUITY_TOLERANCE ) ) &&
 341                         ( sub->start < ( cur->start + PTS_DISCONTINUITY_TOLERANCE ) ) )
 342                     {
 343                         /*
 344                          * The subtitle is from our current time region which we are
 345                          * jumping from. So trash it as we are about to jump backwards
 346                          * or forwards and don't want it blocking the subtitle fifo.
 347                          */
 348                         hb_log("Trashing subtitle 0x%x due to PTS discontinuity", sub);
 349                         sub = hb_fifo_get( pv->subtitle->fifo_raw );
 350                         hb_buffer_close( &sub );
 351                     } else {
 352                         break;
 353                     }
 354                 }
 355             }
 356
 357             /* Trash current picture */
 358             /* Also, make sure we don't trash a chapter break */
 359             chap_break = cur->new_chap;
 360             hb_buffer_close( &cur );
 361             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 362             cur->new_chap |= chap_break; // Don't stomp existing chapter breaks
 363
 364             /* Calculate new offset */
 365             pv->pts_offset_old = pv->pts_offset;
 366             pv->pts_offset     = cur->start -
 367                 pv->count_frames * pv->job->vrate_base / 300;
 368
 369             if( !pv->discontinuity )
 370             {
 371                 pv->discontinuity = 1;
 372             }
 373
 374             pv->video_sequence = cur->sequence;
 375             continue;
 376         }
 377
 378         /* Look for a subtitle for this frame */
 379         if( pv->subtitle )
 380         {
 381             hb_buffer_t * sub2;
 382             while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
 383             {
 384                 /* If two subtitles overlap, make the first one stop
 385                    when the second one starts */
 386                 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
 387                 if( sub2 && sub->stop > sub2->start )
 388                     sub->stop = sub2->start;
 389
 390                 // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 391                 //       sub, cur->sequence, sub->sequence);
 392
 393                 if( sub->sequence > cur->sequence )
 394                 {
 395                     /*
 396                      * The video is behind where we are, so wait until
 397                      * it catches up to the same reader point on the
 398                      * DVD. Then our PTS should be in the same region
 399                      * as the video.
 400                      */
 401                     sub = NULL;
 402                     break;
 403                 }
 404
 405                 if( sub->stop > cur->start ) {
 406                     /*
 407                      * The stop time is in the future, so fall through
 408                      * and we'll deal with it in the next block of
 409                      * code.
 410                      */
 411                     break;
 412                 }
 413                 else
 414                 {
 415                     /*
 416                      * The stop time is in the past. But is it due to
 417                      * it having been played already, or has the PTS
 418                      * been reset to 0?
 419                      */
 420                     if( ( cur->start - sub->stop ) > PTS_DISCONTINUITY_TOLERANCE ) {
 421                         /*
 422                          * There is a lot of time between our current
 423                          * video and where this subtitle is ending,
 424                          * assume that we are about to reset the PTS
 425                          * and do not throw away this subtitle.
 426                          */
 427                         break;
 428                     }
 429                 }
 430
 431                 /*
 432                  * The subtitle is older than this picture, trash it
 433                  */
 434                 sub = hb_fifo_get( pv->subtitle->fifo_raw );
 435                 hb_buffer_close( &sub );
 436             }
 437
 438             /*
 439              * There is a valid subtitle, is it time to display it?
 440              */
 441             if( sub )
 442             {
 443                 if( sub->stop > sub->start)
 444                 {
 445                     /*
 446                      * Normal subtitle which ends after it starts, check to
 447                      * see that the current video is between the start and end.
 448                      */
 449                     if( cur->start > sub->start &&
 450                         cur->start < sub->stop )
 451                     {
 452                         /*
 453                          * We should be playing this, so leave the
 454                          * subtitle in place.
 455                          *
 456                          * fall through to display
 457                          */
 458                     }
 459                     else
 460                     {
 461                         /*
 462                          * Defer until the play point is within the subtitle
 463                          */
 464                         sub = NULL;
 465                     }
 466                 }
 467                 else
 468                 {
 469                     /*
 470                      * The end of the subtitle is less than the start, this is a
 471                      * sign of a PTS discontinuity.
 472                      */
 473                     if( sub->start > cur->start )
 474                     {
 475                         /*
 476                          * we haven't reached the start time yet, or
 477                          * we have jumped backwards after having
 478                          * already started this subtitle.
 479                          */
 480                         if( cur->start < sub->stop )
 481                         {
 482                             /*
 483                              * We have jumped backwards and so should
 484                              * continue displaying this subtitle.
 485                              *
 486                              * fall through to display.
 487                              */
 488                         }
 489                         else
 490                         {
 491                             /*
 492                              * Defer until the play point is within the subtitle
 493                              */
 494                             sub = NULL;
 495                         }
 496                     } else {
 497                         /*
 498                          * Play this subtitle as the start is greater than our
 499                          * video point.
 500                          *
 501                          * fall through to display/
 502                          */
 503                     }
 504                 }
 505             }
 506         }
 507
 508         /* The PTS of the frame we are expecting now */
 509         pts_expected = pv->pts_offset +
 510             pv->count_frames * pv->job->vrate_base / 300;
 511
 512         //hb_log("Video expecting PTS %lld, current frame: %lld, next frame: %lld, cf: %lld",
 513         //       pts_expected, cur->start, next->start, pv->count_frames * pv->job->vrate_base / 300 );
 514
 515         if( cur->start < pts_expected - pv->job->vrate_base / 300 / 2 &&
 516             next->start < pts_expected + pv->job->vrate_base / 300 / 2 )
 517         {
 518             /* The current frame is too old but the next one matches,
 519                let's trash */
 520             /* Also, make sure we don't trash a chapter break */
 521             chap_break = cur->new_chap;
 522             hb_buffer_close( &cur );
 523             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 524             cur->new_chap |= chap_break; // Make sure we don't stomp the existing one.
 525
 526             continue;
 527         }
 528
 529         if( next->start > pts_expected + 3 * pv->job->vrate_base / 300 / 2 )
 530         {
 531             /* We'll need the current frame more than one time. Make a
 532                copy of it and keep it */
 533             buf_tmp = hb_buffer_init( cur->size );
 534             memcpy( buf_tmp->data, cur->data, cur->size );
 535             buf_tmp->sequence = cur->sequence;
 536         }
 537         else
 538         {
 539             /* The frame has the expected date and won't have to be
 540                duplicated, just put it through */
 541             buf_tmp = cur;
 542             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 543         }
 544
 545         /* Replace those MPEG-2 dates with our dates */
 546         buf_tmp->start = (uint64_t) pv->count_frames *
 547             pv->job->vrate_base / 300;
 548         buf_tmp->stop  = (uint64_t) ( pv->count_frames + 1 ) *
 549             pv->job->vrate_base / 300;
 550
 551         /* If we have a subtitle for this picture, copy it */
 552         /* FIXME: we should avoid this memcpy */
 553         if( sub )
 554         {
 555             buf_tmp->sub         = hb_buffer_init( sub->size );
 556             buf_tmp->sub->x      = sub->x;
 557             buf_tmp->sub->y      = sub->y;
 558             buf_tmp->sub->width  = sub->width;
 559             buf_tmp->sub->height = sub->height;
 560             memcpy( buf_tmp->sub->data, sub->data, sub->size );
 561         }
 562
 563         /* Push the frame to the renderer */
 564         hb_fifo_push( job->fifo_sync, buf_tmp );
 565
 566         /* Update UI */
 567         UpdateState( w );
 568
 569         /* Make sure we won't get more frames then expected */
 570         if( pv->count_frames >= pv->count_frames_max * 2)
 571         {
 572             hb_log( "sync: got too many frames (%lld), exiting early", pv->count_frames );
 573             pv->done = 1;
 574
 575            // Drop an empty buffer into our output to ensure that things
 576            // get flushed all the way out.
 577            buf_tmp = hb_buffer_init(0); // Empty end buffer
 578            hb_fifo_push( job->fifo_sync, buf_tmp );
 579
 580             break;
 581         }
 582     }
 583
 584     return HB_WORK_OK;
 585 }
 586
 587 /***********************************************************************
 588  * SyncAudio
 589  ***********************************************************************
 590  *
 591  **********************************************************************/
 592 static void SyncAudio( hb_work_object_t * w, int i )
 593 {
 594     hb_work_private_t * pv = w->private_data;
 595     hb_job_t        * job;
 596     hb_audio_t      * audio;
 597     hb_buffer_t     * buf;
 598     hb_sync_audio_t * sync;
 599
 600     hb_fifo_t       * fifo;
 601     int               rate;
 602
 603     int64_t           pts_expected;
 604     int64_t           start;
 605
 606     job    = pv->job;
 607     sync   = &pv->sync_audio[i];
 608     audio  = sync->audio;
 609
 610     if( job->acodec & HB_ACODEC_AC3 )
 611     {
 612         fifo = audio->fifo_out;
 613         rate = audio->rate;
 614     }
 615     else
 616     {
 617         fifo = audio->fifo_sync;
 618         rate = job->arate;
 619     }
 620
 621     while( !hb_fifo_is_full( fifo ) &&
 622            ( buf = hb_fifo_see( audio->fifo_raw ) ) )
 623     {
 624         /* The PTS of the samples we are expecting now */
 625         pts_expected = pv->pts_offset + sync->count_frames * 90000 / rate;
 626
 627         // hb_log("Video Sequence: %lld, Audio Sequence: %lld", pv->video_sequence, buf->sequence);
 628
 629         /*
 630          * Using the same logic as the Video have we crossed a VOB
 631          * boundary as detected by the expected PTS and the PTS of our
 632          * audio being out by more than the tolerance value.
 633          */
 634         if( buf->start > pts_expected + PTS_DISCONTINUITY_TOLERANCE ||
 635             buf->start < pts_expected - PTS_DISCONTINUITY_TOLERANCE )
 636         {
 637             /* There has been a PTS discontinuity, and this frame might
 638                be from before the discontinuity*/
 639
 640             if( pv->discontinuity )
 641             {
 642                 /*
 643                  * There is an outstanding discontinuity, so use the offset from
 644                  * that discontinuity.
 645                  */
 646                 pts_expected = pv->pts_offset_old + sync->count_frames *
 647                     90000 / rate;
 648             }
 649             else
 650             {
 651                 /*
 652                  * No outstanding discontinuity, so the audio must be leading the
 653                  * video (or the PTS values are really stuffed). So lets mark this
 654                  * as a discontinuity ourselves for the audio to use until
 655                  * the video also crosses the discontinuity.
 656                  *
 657                  * pts_offset is used when we are in the same time space as the video
 658                  * pts_offset_old when in a discontinuity.
 659                  *
 660                  * Therefore set the pts_offset_old given the new pts_offset for this
 661                  * current buffer.
 662                  */
 663                 pv->discontinuity = 1;
 664                 pv->pts_offset_old = buf->start - sync->count_frames *
 665                     90000 / rate;
 666                 pts_expected = pv->pts_offset_old + sync->count_frames *
 667                     90000 / rate;
 668
 669                 hb_log("Sync: Audio discontinuity (sequence: vid %lld aud %lld) (pts %lld < %lld < %lld)",
 670                        pv->video_sequence, buf->sequence,
 671                        pts_expected - PTS_DISCONTINUITY_TOLERANCE, buf->start,
 672                        pts_expected + PTS_DISCONTINUITY_TOLERANCE );
 673             }
 674
 675             /*
 676              * Is the audio from a valid period given the previous
 677              * Video PTS. I.e. has there just been a video PTS
 678              * discontinuity and this audio belongs to the vdeo from
 679              * before?
 680              */
 681             if( buf->start > pts_expected + PTS_DISCONTINUITY_TOLERANCE ||
 682                 buf->start < pts_expected - PTS_DISCONTINUITY_TOLERANCE )
 683             {
 684                 /*
 685                  * It's outside of our tolerance for where the video
 686                  * is now, and it's outside of the tolerance for
 687                  * where we have been in the case of a VOB change.
 688                  * Try and reconverge regardless. so continue on to
 689                  * our convergence code below which will kick in as
 690                  * it will be more than 100ms out.
 691                  *
 692                  * Note that trashing the Audio could make things
 693                  * worse if the Audio is in front because we will end
 694                  * up diverging even more. We need to hold on to the
 695                  * audio until the video catches up.
 696                  */
 697                 if( !pv->way_out_of_sync )
 698                 {
 699                     hb_log("Sync: Audio is way out of sync, attempt to reconverge from current video PTS");
 700                     pv->way_out_of_sync = 1;
 701                 }
 702
 703                 /*
 704                  * It wasn't from the old place, so we must be from
 705                  * the new, but just too far out. So attempt to
 706                  * reconverge by resetting the point we want to be to
 707                  * where we are currently wanting to be.
 708                  */
 709                 pts_expected = pv->pts_offset + sync->count_frames * 90000 / rate;
 710                 start = pts_expected - pv->pts_offset;
 711             } else {
 712                  /* Use the older offset */
 713                 start = pts_expected - pv->pts_offset_old;
 714             }
 715         }
 716         else
 717         {
 718             start = pts_expected - pv->pts_offset;
 719
 720             if( pv->discontinuity )
 721             {
 722                 /*
 723                  * The Audio is tracking the Video again using the normal pts_offset, so the
 724                  * discontinuity is over.
 725                  */
 726                 hb_log( "Sync: Audio joined Video after discontinuity at PTS %lld", buf->start );
 727                 pv->discontinuity = 0;
 728             }
 729         }
 730
 731         /* Tolerance: 100 ms */
 732         if( buf->start < pts_expected - 9000 )
 733         {
 734             if( !pv->trashing_audio )
 735             {
 736                 /* Audio is behind the Video, trash it, can't use it now. */
 737                 hb_log( "Sync: Audio PTS (%lld) < Video PTS (%lld) by greater than 100ms, trashing audio to reconverge",
 738                         buf->start, pts_expected);
 739                 pv->trashing_audio = 1;
 740             }
 741             buf = hb_fifo_get( audio->fifo_raw );
 742             hb_buffer_close( &buf );
 743             continue;
 744         }
 745         else if( buf->start > pts_expected + 9000 )
 746         {
 747             /* Audio is ahead of the Video, insert silence until we catch up*/
 748             if( !pv->inserting_silence )
 749             {
 750                 hb_log("Sync: Audio PTS (%lld) >  Video PTS (%lld) by greater than 100ms insert silence until reconverged", buf->start, pts_expected);
 751                 pv->inserting_silence = 1;
 752             }
 753             InsertSilence( w, i );
 754             continue;
 755         }
 756         else
 757         {
 758             if( pv->trashing_audio || pv->inserting_silence )
 759             {
 760                 hb_log( "Sync: Audio back in Sync at PTS %lld", buf->start );
 761                 pv->trashing_audio = 0;
 762                 pv->inserting_silence = 0;
 763             }
 764             if( pv->way_out_of_sync )
 765             {
 766                 hb_log( "Sync: Audio no longer way out of sync at PTS %lld",
 767                         buf->start );
 768                 pv->way_out_of_sync = 0;
 769             }
 770         }
 771
 772         if( job->acodec & HB_ACODEC_AC3 )
 773         {
 774             buf        = hb_fifo_get( audio->fifo_raw );
 775             buf->start = start;
 776             buf->stop  = start + 90000 * AC3_SAMPLES_PER_FRAME / rate;
 777
 778             sync->count_frames += AC3_SAMPLES_PER_FRAME;
 779         }
 780         else
 781         {
 782             hb_buffer_t * buf_raw = hb_fifo_get( audio->fifo_raw );
 783
 784             int count_in, count_out;
 785
 786             count_in  = buf_raw->size / HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) / sizeof( float );
 787             count_out = ( buf_raw->stop - buf_raw->start ) * job->arate / 90000;
 788             if( buf->start < pts_expected - 1500 )
 789                 count_out--;
 790             else if( buf->start > pts_expected + 1500 )
 791                 count_out++;
 792
 793             sync->data.data_in      = (float *) buf_raw->data;
 794             sync->data.input_frames = count_in;
 795             sync->data.output_frames = count_out;
 796
 797             sync->data.src_ratio = (double) sync->data.output_frames /
 798                                    (double) sync->data.input_frames;
 799
 800             buf = hb_buffer_init( sync->data.output_frames * HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) *
 801                                   sizeof( float ) );
 802             sync->data.data_out = (float *) buf->data;
 803             if( src_process( sync->state, &sync->data ) )
 804             {
 805                 /* XXX If this happens, we're screwed */
 806                 hb_log( "sync: src_process failed" );
 807             }
 808             hb_buffer_close( &buf_raw );
 809
 810             buf->size = sync->data.output_frames_gen * HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) * sizeof( float );
 811
 812             /* Set dates for resampled data */
 813             buf->start = start;
 814             buf->stop  = start + sync->data.output_frames_gen *
 815                             90000 / job->arate;
 816
 817             sync->count_frames += sync->data.output_frames_gen;
 818         }
 819
 820         buf->frametype = HB_FRAME_AUDIO;
 821         hb_fifo_push( fifo, buf );
 822     }
 823
 824     if( hb_fifo_is_full( fifo ) &&
 825         pv->way_out_of_sync )
 826     {
 827         /*
 828          * Trash the top audio packet to avoid dead lock as we reconverge.
 829          */
 830         buf = hb_fifo_get( audio->fifo_raw );
 831         hb_buffer_close( &buf );
 832     }
 833
 834     if( NeedSilence( w, audio ) )
 835     {
 836         InsertSilence( w, i );
 837     }
 838 }
 839
 840 static int NeedSilence( hb_work_object_t * w, hb_audio_t * audio )
 841 {
 842     hb_work_private_t * pv = w->private_data;
 843     hb_job_t * job = pv->job;
 844
 845     if( hb_fifo_size( audio->fifo_in ) ||
 846         hb_fifo_size( audio->fifo_raw ) ||
 847         hb_fifo_size( audio->fifo_sync ) ||
 848         hb_fifo_size( audio->fifo_out ) )
 849     {
 850         /* We have some audio, we are fine */
 851         return 0;
 852     }
 853
 854     /* No audio left in fifos */
 855
 856     if( hb_thread_has_exited( job->reader ) )
 857     {
 858         /* We might miss some audio to complete encoding and muxing
 859            the video track */
 860         hb_log("Reader has exited early, inserting silence.");
 861         return 1;
 862     }
 863
 864     if( hb_fifo_is_full( job->fifo_mpeg2 ) &&
 865         hb_fifo_is_full( job->fifo_raw ) &&
 866         hb_fifo_is_full( job->fifo_sync ) &&
 867         hb_fifo_is_full( job->fifo_render ) &&
 868         hb_fifo_is_full( job->fifo_mpeg4 ) )
 869     {
 870         /* Too much video and no audio, oh-oh */
 871         hb_log("Still got some video - and nothing in the audio fifo, insert silence");
 872         return 1;
 873     }
 874
 875     return 0;
 876 }
 877
 878 static void InsertSilence( hb_work_object_t * w, int i )
 879 {
 880     hb_work_private_t * pv = w->private_data;
 881     hb_job_t        * job;
 882     hb_sync_audio_t * sync;
 883     hb_buffer_t     * buf;
 884
 885     job    = pv->job;
 886     sync   = &pv->sync_audio[i];
 887
 888     if( job->acodec & HB_ACODEC_AC3 )
 889     {
 890         buf        = hb_buffer_init( sync->ac3_size );
 891         buf->start = sync->count_frames * 90000 / sync->audio->rate;
 892         buf->stop  = buf->start + 90000 * AC3_SAMPLES_PER_FRAME /
 893                      sync->audio->rate;
 894         memcpy( buf->data, sync->ac3_buf, buf->size );
 895
 896         hb_log( "sync: adding a silent AC-3 frame for track %x",
 897                 sync->audio->id );
 898         hb_fifo_push( sync->audio->fifo_out, buf );
 899
 900         sync->count_frames += AC3_SAMPLES_PER_FRAME;
 901
 902     }
 903     else
 904     {
 905         buf        = hb_buffer_init( HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->amixdown) * job->arate / 20 *
 906                                      sizeof( float ) );
 907         buf->start = sync->count_frames * 90000 / job->arate;
 908         buf->stop  = buf->start + 90000 / 20;
 909         memset( buf->data, 0, buf->size );
 910
 911         hb_fifo_push( sync->audio->fifo_sync, buf );
 912
 913         sync->count_frames += job->arate / 20;
 914     }
 915 }
 916
 917 static void UpdateState( hb_work_object_t * w )
 918 {
 919     hb_work_private_t * pv = w->private_data;
 920     hb_state_t state;
 921
 922     if( !pv->count_frames )
 923     {
 924         pv->st_first = hb_get_date();
 925     }
 926     pv->count_frames++;
 927
 928     if( hb_get_date() > pv->st_dates[3] + 1000 )
 929     {
 930         memmove( &pv->st_dates[0], &pv->st_dates[1],
 931                  3 * sizeof( uint64_t ) );
 932         memmove( &pv->st_counts[0], &pv->st_counts[1],
 933                  3 * sizeof( uint64_t ) );
 934         pv->st_dates[3]  = hb_get_date();
 935         pv->st_counts[3] = pv->count_frames;
 936     }
 937
 938 #define p state.param.working
 939     state.state = HB_STATE_WORKING;
 940     p.progress  = (float) pv->count_frames / (float) pv->count_frames_max;
 941     if( p.progress > 1.0 )
 942     {
 943         p.progress = 1.0;
 944     }
 945     p.rate_cur   = 1000.0 *
 946         (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
 947         (float) ( pv->st_dates[3] - pv->st_dates[0] );
 948     if( hb_get_date() > pv->st_first + 4000 )
 949     {
 950         int eta;
 951         p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
 952             (float) ( pv->st_dates[3] - pv->st_first );
 953         eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
 954             p.rate_avg;
 955         p.hours   = eta / 3600;
 956         p.minutes = ( eta % 3600 ) / 60;
 957         p.seconds = eta % 60;
 958     }
 959     else
 960     {
 961         p.rate_avg = 0.0;
 962         p.hours    = -1;
 963         p.minutes  = -1;
 964         p.seconds  = -1;
 965     }
 966 #undef p
 967
 968     hb_set_state( pv->job->h, &state );
 969 }