libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.fr/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8 #include "hbffmpeg.h"
   9 #include <stdio.h>
  10 #include "samplerate.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_lock_t * mutex;
  22     int         ref;        /* Reference count to tell us when it's unused */
  23     int         count_frames;
  24     int64_t     audio_pts_slip;
  25     int64_t     video_pts_slip;
  26     int64_t     pts_offset;
  27
  28     /* Frame based point-to-point support */
  29     int64_t     audio_pts_thresh;
  30     int         start_found;
  31     hb_cond_t * next_frame;
  32     int         pts_count;
  33     int64_t   * first_pts;
  34 } hb_sync_common_t;
  35
  36 typedef struct
  37 {
  38     int          index;
  39     double       next_start;   /* start time of next output frame */
  40     int64_t      first_drop;   /* PTS of first 'went backwards' frame dropped */
  41     int          drop_count;   /* count of 'time went backwards' drops */
  42
  43     /* Raw */
  44     SRC_STATE  * state;
  45     SRC_DATA     data;
  46
  47     /* AC-3 */
  48     int          ac3_size;
  49     uint8_t    * ac3_buf;
  50 } hb_sync_audio_t;
  51
  52 typedef struct
  53 {
  54     /* Video */
  55     int        first_frame;
  56     int64_t    pts_skip;
  57     int64_t    next_start;    /* start time of next output frame */
  58     int64_t    first_drop;    /* PTS of first 'went backwards' frame dropped */
  59     int        drop_count;    /* count of 'time went backwards' drops */
  60     int        drops;         /* frames dropped to make a cbr video stream */
  61     int        dups;          /* frames duplicated to make a cbr video stream */
  62     int        video_sequence;
  63     int        count_frames_max;
  64     int        chap_mark;     /* to propagate chapter mark across a drop */
  65     hb_buffer_t * cur;        /* The next picture to process */
  66
  67     /* Statistics */
  68     uint64_t   st_counts[4];
  69     uint64_t   st_dates[4];
  70     uint64_t   st_first;
  71
  72     /* Subtitles */
  73     hb_buffer_t * sub_list;   /* list of subtitles to be passed thru or rendered */
  74 } hb_sync_video_t;
  75
  76 struct hb_work_private_s
  77 {
  78     hb_job_t * job;
  79     hb_sync_common_t * common;
  80     union
  81     {
  82         hb_sync_video_t video;
  83         hb_sync_audio_t audio;
  84     } type;
  85 };
  86
  87 /***********************************************************************
  88  * Local prototypes
  89  **********************************************************************/
  90 static void getPtsOffset( hb_work_object_t * w );
  91 static int  checkPtsOffset( hb_work_object_t * w );
  92 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i );
  93 static void InsertSilence( hb_work_object_t * w, int64_t d );
  94 static void UpdateState( hb_work_object_t * w );
  95 static void UpdateSearchState( hb_work_object_t * w, int64_t start );
  96 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
  97                                        hb_sync_audio_t *sync );
  98
  99 /***********************************************************************
 100  * hb_work_sync_init
 101  ***********************************************************************
 102  * Initialize the work object
 103  **********************************************************************/
 104 hb_work_object_t * hb_sync_init( hb_job_t * job )
 105 {
 106     hb_title_t        * title = job->title;
 107     hb_chapter_t      * chapter;
 108     int                 i;
 109     uint64_t            duration;
 110     hb_work_private_t * pv;
 111     hb_sync_video_t   * sync;
 112     hb_work_object_t  * w;
 113     hb_work_object_t  * ret = NULL;
 114
 115     pv = calloc( 1, sizeof( hb_work_private_t ) );
 116     sync = &pv->type.video;
 117     pv->common = calloc( 1, sizeof( hb_sync_common_t ) );
 118     pv->common->ref++;
 119     pv->common->mutex = hb_lock_init();
 120     pv->common->audio_pts_thresh = -1;
 121     pv->common->next_frame = hb_cond_init();
 122     pv->common->pts_count = 1;
 123     if ( job->frame_to_start || job->pts_to_start )
 124     {
 125         pv->common->start_found = 0;
 126     }
 127     else
 128     {
 129         pv->common->start_found = 1;
 130     }
 131
 132     ret = w = hb_get_work( WORK_SYNC_VIDEO );
 133     w->private_data = pv;
 134     w->fifo_in = job->fifo_raw;
 135     w->fifo_out = job->fifo_sync;
 136
 137     pv->job            = job;
 138     pv->common->pts_offset   = INT64_MIN;
 139     sync->first_frame = 1;
 140
 141     if( job->pass == 2 )
 142     {
 143         /* We already have an accurate frame count from pass 1 */
 144         hb_interjob_t * interjob = hb_interjob_get( job->h );
 145         sync->count_frames_max = interjob->frame_count;
 146     }
 147     else
 148     {
 149         /* Calculate how many video frames we are expecting */
 150         if ( job->pts_to_stop )
 151         {
 152             duration = job->pts_to_stop + 90000;
 153         }
 154         else if( job->frame_to_stop )
 155         {
 156             /* Set the duration to a rough estimate */
 157             duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
 158         }
 159         else
 160         {
 161             duration = 0;
 162             for( i = job->chapter_start; i <= job->chapter_end; i++ )
 163             {
 164                 chapter   = hb_list_item( title->list_chapter, i - 1 );
 165                 duration += chapter->duration;
 166             }
 167             duration += 90000;
 168             /* 1 second safety so we're sure we won't miss anything */
 169         }
 170         sync->count_frames_max = duration * title->rate / title->rate_base / 90000;
 171     }
 172
 173     hb_log( "sync: expecting %d video frames", sync->count_frames_max );
 174
 175     /* Initialize libsamplerate for every audio track we have */
 176     if ( ! job->indepth_scan )
 177     {
 178         for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 179         {
 180             InitAudio( job, pv->common, i );
 181         }
 182     }
 183     pv->common->first_pts = malloc( sizeof(int64_t) * pv->common->pts_count );
 184     for ( i = 0; i < pv->common->pts_count; i++ )
 185         pv->common->first_pts[i] = INT64_MAX;
 186
 187     return ret;
 188 }
 189
 190 /***********************************************************************
 191  * Close Video
 192  ***********************************************************************
 193  *
 194  **********************************************************************/
 195 void syncVideoClose( hb_work_object_t * w )
 196 {
 197     hb_work_private_t * pv = w->private_data;
 198     hb_job_t          * job   = pv->job;
 199     hb_sync_video_t   * sync = &pv->type.video;
 200
 201     // Wake up audio sync if it's still waiting on condition.
 202     pv->common->pts_offset = 0;
 203     pv->common->start_found = 1;
 204     hb_cond_broadcast( pv->common->next_frame );
 205
 206     if( sync->cur )
 207     {
 208         hb_buffer_close( &sync->cur );
 209     }
 210
 211     hb_log( "sync: got %d frames, %d expected",
 212             pv->common->count_frames, sync->count_frames_max );
 213
 214     /* save data for second pass */
 215     if( job->pass == 1 )
 216     {
 217         /* Preserve frame count for better accuracy in pass 2 */
 218         hb_interjob_t * interjob = hb_interjob_get( job->h );
 219         interjob->frame_count = pv->common->count_frames;
 220         interjob->last_job = job->sequence_id;
 221         interjob->total_time = sync->next_start;
 222     }
 223
 224     if (sync->drops || sync->dups )
 225     {
 226         hb_log( "sync: %d frames dropped, %d duplicated",
 227                 sync->drops, sync->dups );
 228     }
 229
 230     hb_lock( pv->common->mutex );
 231     if ( --pv->common->ref == 0 )
 232     {
 233         hb_unlock( pv->common->mutex );
 234         hb_lock_close( &pv->common->mutex );
 235         free( pv->common );
 236     }
 237     else
 238     {
 239         hb_unlock( pv->common->mutex );
 240     }
 241
 242     free( pv );
 243     w->private_data = NULL;
 244 }
 245
 246 /***********************************************************************
 247  * syncVideoWork
 248  ***********************************************************************
 249  *
 250  **********************************************************************/
 251 static hb_buffer_t * copy_subtitle( hb_buffer_t * src );
 252
 253 int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
 254               hb_buffer_t ** buf_out )
 255 {
 256     hb_buffer_t * cur, * next, * sub = NULL;
 257     hb_work_private_t * pv = w->private_data;
 258     hb_job_t          * job = pv->job;
 259     hb_subtitle_t     * subtitle;
 260     hb_sync_video_t   * sync = &pv->type.video;
 261     int i;
 262     int64_t start, next_start;
 263
 264     *buf_out = NULL;
 265     next = *buf_in;
 266     *buf_in = NULL;
 267
 268     /* Wait till we can determine the initial pts of all streams */
 269     if( next->size != 0 && pv->common->pts_offset == INT64_MIN )
 270     {
 271         pv->common->first_pts[0] = next->start;
 272         hb_lock( pv->common->mutex );
 273         while( pv->common->pts_offset == INT64_MIN )
 274         {
 275             // Full fifos will make us wait forever, so get the
 276             // pts offset from the available streams if full
 277             if ( hb_fifo_is_full( job->fifo_raw ) )
 278             {
 279                 getPtsOffset( w );
 280                 hb_cond_broadcast( pv->common->next_frame );
 281             }
 282             else if ( checkPtsOffset( w ) )
 283                 hb_cond_broadcast( pv->common->next_frame );
 284             else
 285                 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
 286         }
 287         hb_unlock( pv->common->mutex );
 288     }
 289
 290     hb_lock( pv->common->mutex );
 291     next_start = next->start - pv->common->video_pts_slip;
 292     hb_unlock( pv->common->mutex );
 293
 294     /* Wait for start of point-to-point encoding */
 295     if( !pv->common->start_found )
 296     {
 297         hb_sync_video_t   * sync = &pv->type.video;
 298
 299         if( next->size == 0 )
 300         {
 301             *buf_out = next;
 302             pv->common->start_found = 1;
 303             pv->common->first_pts[0] = INT64_MAX - 1;
 304             hb_cond_broadcast( pv->common->next_frame );
 305
 306             /*
 307              * Push through any subtitle EOFs in case they
 308              * were not synced through.
 309              */
 310             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 311             {
 312                 subtitle = hb_list_item( job->list_subtitle, i );
 313                 if( subtitle->config.dest == PASSTHRUSUB )
 314                 {
 315                     if( subtitle->source == VOBSUB )
 316                         hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
 317                     else
 318                         hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 319                 }
 320             }
 321             return HB_WORK_DONE;
 322         }
 323         if ( pv->common->count_frames < job->frame_to_start ||
 324              next->start < job->pts_to_start )
 325         {
 326             // Flush any subtitles that have pts prior to the
 327             // current frame
 328             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 329             {
 330                 subtitle = hb_list_item( job->list_subtitle, i );
 331                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 332                 {
 333                     if ( sub->start > next->start )
 334                         break;
 335                     sub = hb_fifo_get( subtitle->fifo_raw );
 336                     hb_buffer_close( &sub );
 337                 }
 338             }
 339             hb_lock( pv->common->mutex );
 340             // Tell the audio threads what must be dropped
 341             pv->common->audio_pts_thresh = next_start + pv->common->video_pts_slip;
 342             hb_cond_broadcast( pv->common->next_frame );
 343             hb_unlock( pv->common->mutex );
 344
 345             UpdateSearchState( w, next_start );
 346             hb_buffer_close( &next );
 347
 348             return HB_WORK_OK;
 349         }
 350         hb_lock( pv->common->mutex );
 351         pv->common->audio_pts_thresh = 0;
 352         pv->common->audio_pts_slip += next_start;
 353         pv->common->video_pts_slip += next_start;
 354         next_start = 0;
 355         pv->common->start_found = 1;
 356         pv->common->count_frames = 0;
 357         hb_cond_broadcast( pv->common->next_frame );
 358         hb_unlock( pv->common->mutex );
 359         sync->st_first = 0;
 360     }
 361
 362     if( !sync->cur )
 363     {
 364         sync->cur = next;
 365         if( sync->cur->size == 0 )
 366         {
 367             /* we got an end-of-stream as our first video packet?
 368              * Feed it downstream & signal that we're done.
 369              */
 370             *buf_out = hb_buffer_init( 0 );
 371
 372             pv->common->start_found = 1;
 373             pv->common->first_pts[0] = INT64_MAX - 1;
 374             hb_cond_broadcast( pv->common->next_frame );
 375
 376             /*
 377              * Push through any subtitle EOFs in case they
 378              * were not synced through.
 379              */
 380             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 381             {
 382                 subtitle = hb_list_item( job->list_subtitle, i );
 383                 if( subtitle->config.dest == PASSTHRUSUB )
 384                 {
 385                     if( subtitle->source == VOBSUB )
 386                         hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
 387                     else
 388                         hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 389                 }
 390             }
 391             return HB_WORK_DONE;
 392         }
 393         return HB_WORK_OK;
 394     }
 395     cur = sync->cur;
 396     /* At this point we have a frame to process. Let's check
 397         1) if we will be able to push into the fifo ahead
 398         2) if the next frame is there already, since we need it to
 399            compute the duration of the current frame*/
 400     if( next->size == 0 )
 401     {
 402         hb_buffer_close( &next );
 403
 404         pv->common->first_pts[0] = INT64_MAX - 1;
 405         cur->start = sync->next_start;
 406         cur->stop = cur->start + 90000. / ((double)job->vrate / (double)job->vrate_base);
 407
 408         /* Make sure last frame is reflected in frame count */
 409         pv->common->count_frames++;
 410
 411         /* Push the frame to the renderer */
 412         hb_fifo_push( job->fifo_sync, cur );
 413         sync->cur = NULL;
 414
 415         /* we got an end-of-stream. Feed it downstream & signal that
 416          * we're done. Note that this means we drop the final frame of
 417          * video (we don't know its duration). On DVDs the final frame
 418          * is often strange and dropping it seems to be a good idea. */
 419         *buf_out = hb_buffer_init( 0 );
 420
 421         /*
 422          * Push through any subtitle EOFs in case they were not synced through.
 423          */
 424         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 425         {
 426             subtitle = hb_list_item( job->list_subtitle, i );
 427             if( subtitle->config.dest == PASSTHRUSUB )
 428             {
 429                 if( subtitle->source == VOBSUB )
 430                     hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
 431                 else
 432                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 433             }
 434         }
 435         pv->common->start_found = 1;
 436         hb_cond_broadcast( pv->common->next_frame );
 437         return HB_WORK_DONE;
 438     }
 439
 440     /* Check for end of point-to-point frame encoding */
 441     if( job->frame_to_stop && pv->common->count_frames > job->frame_to_stop )
 442     {
 443         // Drop an empty buffer into our output to ensure that things
 444         // get flushed all the way out.
 445         hb_buffer_close( &sync->cur );
 446         hb_buffer_close( &next );
 447         *buf_out = hb_buffer_init( 0 );
 448         hb_log( "sync: reached %d frames, exiting early",
 449                 pv->common->count_frames );
 450
 451         /*
 452          * Push through any subtitle EOFs in case they were not synced through.
 453          */
 454         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 455         {
 456             subtitle = hb_list_item( job->list_subtitle, i );
 457             if( subtitle->config.dest == PASSTHRUSUB )
 458             {
 459                 if( subtitle->source == VOBSUB )
 460                     hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
 461                 else
 462                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 463             }
 464         }
 465         return HB_WORK_DONE;
 466     }
 467
 468     hb_lock( pv->common->mutex );
 469     start = cur->start - pv->common->video_pts_slip;
 470     hb_unlock( pv->common->mutex );
 471
 472     /* Check for end of point-to-point pts encoding */
 473     if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
 474     {
 475         // Drop an empty buffer into our output to ensure that things
 476         // get flushed all the way out.
 477         hb_log( "sync: reached pts %"PRId64", exiting early", start );
 478         hb_buffer_close( &sync->cur );
 479         hb_buffer_close( &next );
 480         *buf_out = hb_buffer_init( 0 );
 481
 482         /*
 483          * Push through any subtitle EOFs in case they were not synced through.
 484          */
 485         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 486         {
 487             subtitle = hb_list_item( job->list_subtitle, i );
 488             if( subtitle->config.dest == PASSTHRUSUB )
 489             {
 490                 if( subtitle->source == VOBSUB )
 491                     hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
 492                 else
 493                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 494             }
 495         }
 496         return HB_WORK_DONE;
 497     }
 498
 499     if( sync->first_frame )
 500     {
 501         /* This is our first frame */
 502         if ( start > 0 )
 503         {
 504             /*
 505              * The first pts from a dvd should always be zero but
 506              * can be non-zero with a transport or program stream since
 507              * we're not guaranteed to start on an IDR frame. If we get
 508              * a non-zero initial PTS extend its duration so it behaves
 509              * as if it started at zero so that our audio timing will
 510              * be in sync.
 511              */
 512             hb_log( "sync: first pts is %"PRId64, start );
 513             start = 0;
 514         }
 515         sync->first_frame = 0;
 516     }
 517
 518     /*
 519      * since the first frame is always 0 and the upstream reader code
 520      * is taking care of adjusting for pts discontinuities, we just have
 521      * to deal with the next frame's start being in the past. This can
 522      * happen when the PTS is adjusted after data loss but video frame
 523      * reordering causes some frames with the old clock to appear after
 524      * the clock change. This creates frames that overlap in time which
 525      * looks to us like time going backward. The downstream muxing code
 526      * can deal with overlaps of up to a frame time but anything larger
 527      * we handle by dropping frames here.
 528      */
 529     if ( next_start - start <= 0 )
 530     {
 531         if ( sync->first_drop == 0 )
 532         {
 533             sync->first_drop = next_start;
 534         }
 535         ++sync->drop_count;
 536         if ( next->new_chap )
 537         {
 538             // don't drop a chapter mark when we drop the buffer
 539             sync->chap_mark = next->new_chap;
 540         }
 541         hb_buffer_close( &next );
 542         return HB_WORK_OK;
 543     }
 544     if ( sync->first_drop )
 545     {
 546         hb_log( "sync: video time didn't advance - dropped %d frames "
 547                 "(delta %d ms, current %"PRId64", next %"PRId64", dur %d)",
 548                 sync->drop_count, (int)( start - sync->first_drop ) / 90,
 549                 start, next_start, (int)( next_start - start ) );
 550         sync->first_drop = 0;
 551         sync->drop_count = 0;
 552     }
 553
 554     /*
 555      * Track the video sequence number locally so that we can sync the audio
 556      * to it using the sequence number as well as the PTS.
 557      */
 558     sync->video_sequence = cur->sequence;
 559
 560     /* Process subtitles that apply to this video frame */
 561
 562     // NOTE: There is no logic in either subtitle-sync algorithm that waits for the
 563     //       subtitle-decoder if it is lagging behind the video-decoder.
 564     //
 565     //       Therefore there is the implicit assumption that the subtitle-decoder
 566     //       is always faster than the video-decoder. This assumption is definitely
 567     //       incorrect in some cases where the SSA subtitle decoder is used.
 568     //       Enable the SUBSYNC_VERBOSE_TIMING flag below to debug.
 569
 570 #define SUBSYNC_ALGORITHM_SIMULTANEOUS 1
 571 #define SUBSYNC_ALGORITHM_CLASSIC 0
 572
 573 /*
 574  * Enables logging of three kinds of events:
 575  *   SUB***: Subtitle received by sync object
 576  *   SUB+++: Subtitle now shown
 577  *   SUB---: Subtitle now hidden and disposed
 578  *
 579  * Lead times on SUB*** events should be positive.
 580  *   Negative lead times lead to lag times on SUB+++ or the complete drop of a subtitle.
 581  * Lag times on SUB+++ and SUB--- should be small positive values in the 0-40ms range.
 582  */
 583 #define SUBSYNC_VERBOSE_TIMING 0
 584
 585 #if SUBSYNC_ALGORITHM_SIMULTANEOUS
 586     #define sub_list sync->sub_list
 587     /*
 588      * 1. Find all subtitles that need to be burned into the current video frame
 589      *    and attach them to the frame.
 590      * 2. Find all subtitles that need to be passed thru and do so immediately.
 591      */
 592     for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 593     {
 594         subtitle = hb_list_item( job->list_subtitle, i );
 595
 596         // If this subtitle track's packets are to be passed thru, do so immediately
 597         if( subtitle->config.dest == PASSTHRUSUB )
 598         {
 599             while ( ( sub = hb_fifo_get( subtitle->fifo_raw ) ) != NULL )
 600             {
 601                 if ( subtitle->source == VOBSUB )
 602                 {
 603                     hb_fifo_push( subtitle->fifo_sync, sub );
 604                 }
 605                 else
 606                 {
 607                     hb_fifo_push( subtitle->fifo_out, sub );
 608                 }
 609             }
 610         }
 611         // If this subtitle track's packets are to be rendered, identify the
 612         // packets that need to be rendered on the current video frame
 613         else if( subtitle->config.dest == RENDERSUB )
 614         {
 615             // Migrate subtitles from 'subtitle->fifo_raw' to 'sub_list' immediately.
 616             // Note that the size of 'sub_list' is unbounded.
 617             while ( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) != NULL )
 618             {
 619                 sub = hb_fifo_get( subtitle->fifo_raw );  // pop
 620
 621                 #if SUBSYNC_VERBOSE_TIMING
 622                     printf( "\nSUB*** (%"PRId64"/%"PRId64":%"PRId64") @ %"PRId64"/%"PRId64":%"PRId64" (lead by %"PRId64"ms)\n",
 623                         sub->start/90, sub->start/90/1000/60, sub->start/90/1000%60,
 624                         cur->start/90, cur->start/90/1000/60, cur->start/90/1000%60,
 625                         (sub->start - cur->start)/90);
 626                     if (pv->common->video_pts_slip)
 627                     {
 628                         printf( "  VIDEO-LAG: %"PRId64"\n", pv->common->video_pts_slip );
 629                     }
 630                 #endif
 631
 632                 // Prepend to sub_list
 633                 hb_buffer_t *sub_list_next = sub_list;
 634                 sub_list = sub;
 635                 sub_list->next = sub_list_next;
 636             }
 637
 638             hb_buffer_t *last_sub = NULL;
 639             for ( sub = sub_list; sub != NULL; )
 640             {
 641                 // NOTE: Strictly speaking this sequence check is probably unnecessary.
 642                 //       It is a holdover behavior inherited from the classic subsync algorithm.
 643                 if ( sub->sequence > cur->sequence )
 644                 {
 645                     // Subtitle sequence in the future
 646
 647                     // (Keep the subtitle in the stream)
 648                     last_sub = sub;
 649                     sub = sub->next;
 650                     continue;
 651                 }
 652
 653                 if ( cur->start < sub->start )
 654                 {
 655                     // Subtitle starts in the future
 656
 657                     // (Keep the subtitle in the stream)
 658                     last_sub = sub;
 659                     sub = sub->next;
 660                     continue;
 661                 }
 662                 else
 663                 {
 664                     // Subtitle starts in the past...
 665
 666                     if ( cur->start < sub->stop )
 667                     {
 668                         // Subtitle starts in the past and finishes in the future
 669
 670                         // Attach a copy of the subtitle packet to the current video packet
 671                         // to be burned in by the 'render' work-object.
 672                         // (Can't just alias it because we don't know when the 'render'
 673                         //  work-object will dispose of it.)
 674                         hb_buffer_t * old_sublist_head = cur->sub;
 675                         cur->sub = copy_subtitle( sub );
 676                         cur->sub->next = old_sublist_head;
 677
 678                         #if SUBSYNC_VERBOSE_TIMING
 679                             if (!(sub->new_chap & 0x01))
 680                             {
 681                                 printf( "\nSUB+++ (%"PRId64"/%"PRId64":%"PRId64") @ %"PRId64"/%"PRId64":%"PRId64" (lag by %"PRId64"ms)\n",
 682                                     sub->start/90, sub->start/90/1000/60, sub->start/90/1000%60,
 683                                     cur->start/90, cur->start/90/1000/60, cur->start/90/1000%60,
 684                                     (cur->start - sub->start)/90 );
 685                                 if (pv->common->video_pts_slip)
 686                                 {
 687                                     printf( "  VIDEO-LAG: %"PRId64"\n", pv->common->video_pts_slip );
 688                                 }
 689
 690                                 sub->new_chap |= 0x01;
 691                             }
 692                         #endif
 693
 694                         // (Keep the subtitle in the stream)
 695                         last_sub = sub;
 696                         sub = sub->next;
 697                         continue;
 698                     }
 699                     else
 700                     {
 701                         // Subtitle starts in the past and has already finished
 702
 703                         #if SUBSYNC_VERBOSE_TIMING
 704                             printf( "\nSUB--- (%"PRId64"/%"PRId64":%"PRId64") @ %"PRId64"/%"PRId64":%"PRId64" (lag by %"PRId64"ms)\n",
 705                                 sub->start/90, sub->start/90/1000/60, sub->start/90/1000%60,
 706                                 cur->start/90, cur->start/90/1000/60, cur->start/90/1000%60,
 707                                 (cur->start - sub->stop)/90 );
 708                             if (pv->common->video_pts_slip)
 709                             {
 710                                 printf( "  VIDEO-LAG: %"PRId64"\n", pv->common->video_pts_slip );
 711                             }
 712                         #endif
 713
 714                         // Remove it from the stream...
 715                         if (last_sub != NULL)
 716                         {
 717                             last_sub->next = sub->next;
 718                         }
 719                         if (sub_list == sub)
 720                         {
 721                             sub_list = sub->next;
 722                         }
 723
 724                         // ...and trash it
 725                         hb_buffer_t *next_sub = sub->next;
 726                         // XXX: Prevent hb_buffer_close from killing the whole list
 727                         //      before we finish iterating over it
 728                         sub->next = NULL;
 729
 730                         hb_buffer_t * subpicture_list = sub;
 731                         hb_buffer_t * subpicture;
 732                         hb_buffer_t * subpicture_next;
 733                         for ( subpicture = subpicture_list; subpicture; subpicture = subpicture_next )
 734                         {
 735                             subpicture_next = subpicture->next_subpicture;
 736
 737                             hb_buffer_close( &subpicture );
 738                         }
 739
 740                         // (last_sub remains the same)
 741                         sub = next_sub;
 742                         continue;
 743                     }
 744                 }
 745             }
 746         }
 747     } // end subtitles
 748     #undef sub_list
 749
 750 #elif SUBSYNC_ALGORITHM_CLASSIC
 751     // NOTE: This algorithm does not correctly support the simultaneous display of temporally overlapping subtitles.
 752
 753     /*
 754      * Look for a subtitle for this frame.
 755      *
 756      * If found then it will be tagged onto a video buffer of the correct time and
 757      * sent in to the render pipeline. This only needs to be done for VOBSUBs which
 758      * get rendered, other types of subtitles can just sit in their raw_queue until
 759      * delt with at muxing.
 760      */
 761     for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 762     {
 763         int64_t sub_start, sub_stop, duration;
 764         subtitle = hb_list_item( job->list_subtitle, i );
 765
 766         /*
 767          * Rewrite timestamps on subtitles that need it (on raw queue).
 768          */
 769         // NOTE: It's probably fine to use this logic for passthru VOBSUBs as well,
 770         //       but I am currently preserving backwards compatibility with the old
 771         //       VOBSUB behavior, which uses the more complex logic following this if-statement.
 772         if( subtitle->config.dest == PASSTHRUSUB && subtitle->source != VOBSUB )
 773         {
 774             /*
 775              * Rewrite timestamps on subtitles that came from Closed Captions
 776              * since they are using the MPEG2 timestamps.
 777              */
 778             while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 779             {
 780                 hb_lock( pv->common->mutex );
 781                 sub_start = sub->start - pv->common->video_pts_slip;
 782                 hb_unlock( pv->common->mutex );
 783                 duration = sub->stop - sub->start;
 784                 sub_stop = sub_start + duration;
 785
 786                 /*
 787                  * Rewrite the timestamps as and when the video
 788                  * (cur->start) reaches the same timestamp as a
 789                  * closed caption (sub->start).
 790                  *
 791                  * What about discontinuity boundaries - not delt
 792                  * with here - Van?
 793                  *
 794                  * Bypass the sync fifo altogether.
 795                  */
 796                 if( sub->size <= 0 )
 797                 {
 798                     sub = hb_fifo_get( subtitle->fifo_raw );
 799                     hb_fifo_push( subtitle->fifo_out, sub );
 800                     sub = NULL;
 801                     break;
 802                 } else {
 803                     /*
 804                      * Sync the subtitles to the incoming video, and use
 805                      * the matching converted video timestamp.
 806                      *
 807                      * Note that it doesn't appear that we need to convert
 808                      * timestamps, I guess that they were already correct,
 809                      * so just push them through for rendering.
 810                      *
 811                      */
 812                     if( sub_start <= start )
 813                     {
 814                         sub = hb_fifo_get( subtitle->fifo_raw );
 815                         sub->start = sub_start;
 816                         sub->stop = sub_stop;
 817                         hb_fifo_push( subtitle->fifo_out, sub );
 818                     } else {
 819                         // sub too early. Leave it in the fifo.
 820                         sub = NULL;
 821                         break;
 822                     }
 823                 }
 824             }
 825
 826             continue;
 827         }
 828
 829         // For rendered subtitles (and, for backward compatibility, passthru VOBSUBs),
 830         // delay pushing subtitle packets through the pipeline until the video catches up
 831         if( subtitle->config.dest == RENDERSUB || subtitle->source == VOBSUB )
 832         {
 833             hb_buffer_t * sub2;
 834             while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 835             {
 836                 if( sub->size == 0 )
 837                 {
 838                     /*
 839                      * EOF, pass it through immediately.
 840                      */
 841                     break;
 842                 }
 843
 844                 hb_lock( pv->common->mutex );
 845                 sub_start = sub->start - pv->common->video_pts_slip;
 846                 hb_unlock( pv->common->mutex );
 847                 duration = sub->stop - sub->start;
 848                 sub_stop = sub_start + duration;
 849
 850                 /* If two DVD subtitles overlap, make the first one stop
 851                    when the second one starts */
 852                 // TODO: Consider removing this entirely. Currently retained
 853                 //       to preserve old DVD subtitle behavior.
 854                 if ( subtitle->source == VOBSUB )
 855                 {
 856                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 857                     if( sub2 && sub->stop > sub2->start )
 858                     {
 859                         sub->stop = sub2->start;
 860                     }
 861                 }
 862
 863
 864                 // hb_log("0x%x: video seq: %"PRId64" subtitle sequence: %"PRId64,
 865                 //       sub, cur->sequence, sub->sequence);
 866
 867                 if( sub->sequence > cur->sequence )
 868                 {
 869                     /*
 870                      * The video is behind where we are, so wait until
 871                      * it catches up to the same reader point on the
 872                      * DVD. Then our PTS should be in the same region
 873                      * as the video.
 874                      */
 875                     sub = NULL;
 876                     break;
 877                 }
 878
 879                 #if SUBSYNC_VERBOSE_TIMING
 880                     if (!(sub->new_chap & 0x02))
 881                     {
 882                         printf( "\nSUB*** (%"PRId64"/%"PRId64":%"PRId64") @ %"PRId64"/%"PRId64":%"PRId64" (lead by %"PRId64"ms)\n",
 883                             sub->start/90, sub->start/90/1000/60, sub->start/90/1000%60,
 884                             cur->start/90, cur->start/90/1000/60, cur->start/90/1000%60,
 885                             (sub->start - cur->start)/90);
 886
 887                         sub->new_chap |= 0x02;
 888                     }
 889                 #endif
 890
 891                 if( sub_stop > start )
 892                 {
 893                     // CONDITION: cur->start < sub->stop
 894
 895                     /*
 896                      * The stop time is in the future, so fall through
 897                      * and we'll deal with it in the next block of
 898                      * code.
 899                      */
 900
 901                     /*
 902                      * There is a valid subtitle, is it time to display it?
 903                      */
 904                     if( sub_stop > sub_start)
 905                     {
 906                         // CONDITION: {cur->start, sub->start} < sub->stop
 907
 908                         /*
 909                          * Normal subtitle which ends after it starts,
 910                          * check to see that the current video is between
 911                          * the start and end.
 912                          */
 913                         if( start > sub_start &&
 914                             start < sub_stop )
 915                         {
 916                             // CONDITION: sub->start < cur->start < sub->stop
 917
 918                             /*
 919                             * We should be playing this, so leave the
 920                             * subtitle in place.
 921                             *
 922                             * fall through to display
 923                             */
 924                         }
 925                         else
 926                         {
 927                             // CONDITION: cur->start < sub->start < sub->stop
 928
 929                             /*
 930                              * Defer until the play point is within
 931                              * the subtitle
 932                              */
 933                             sub = NULL;
 934                         }
 935                     }
 936                     else
 937                     {
 938                         // CONDITION: cur->start < sub->stop < sub->start
 939
 940                         /*
 941                          * The end of the subtitle is less than the start,
 942                          * this is a sign of a PTS discontinuity.
 943                          */
 944                         if( sub_start > start )
 945                         {
 946                             // CONDITION: cur->start < sub->stop < sub->start
 947
 948                             /*
 949                              * we haven't reached the start time yet, or
 950                              * we have jumped backwards after having
 951                              * already started this subtitle.
 952                              */
 953                             if( start < sub_stop )
 954                             {
 955                                 // CONDITION: cur->start < sub->stop < sub->start
 956
 957                                 /*
 958                                  * We have jumped backwards and so should
 959                                  * continue displaying this subtitle.
 960                                  *
 961                                  * fall through to display.
 962                                  */
 963                             }
 964                             else
 965                             {
 966                                 // CONDITION: Mathematically impossible to get here
 967
 968                                 /*
 969                                  * Defer until the play point is
 970                                  * within the subtitle
 971                                  */
 972                                 sub = NULL;
 973                             }
 974                         } else {
 975                             // CONDITION: Mathematically impossible to get here
 976
 977                             /*
 978                             * Play this subtitle as the start is
 979                             * greater than our video point.
 980                             *
 981                             * fall through to display/
 982                             */
 983                         }
 984                     }
 985                         break;
 986                 }
 987                 else
 988                 {
 989                     // CONDITION: sub->stop < cur->start
 990
 991                     #if SUBSYNC_VERBOSE_TIMING
 992                         printf( "\nSUB--- (%"PRId64"/%"PRId64":%"PRId64") @ %"PRId64"/%"PRId64":%"PRId64" (lag by %"PRId64"ms)\n",
 993                             sub->start/90, sub->start/90/1000/60, sub->start/90/1000%60,
 994                             cur->start/90, cur->start/90/1000/60, cur->start/90/1000%60,
 995                             (cur->start - sub->stop)/90 );
 996                     #endif
 997
 998                     /*
 999                      * The subtitle is older than this picture, trash it
1000                      */
1001                     sub = hb_fifo_get( subtitle->fifo_raw );
1002                     hb_buffer_close( &sub );
1003                 }
1004             }
1005
1006             /* If we have a subtitle for this picture, copy it */
1007             if( sub )
1008             {
1009                 #if SUBSYNC_VERBOSE_TIMING
1010                     if (!(sub->new_chap & 0x01))
1011                     {
1012                         printf( "\nSUB+++ (%"PRId64"/%"PRId64":%"PRId64") @ %"PRId64"/%"PRId64":%"PRId64" (lag by %"PRId64"ms)\n",
1013                             sub->start/90, sub->start/90/1000/60, sub->start/90/1000%60,
1014                             cur->start/90, cur->start/90/1000/60, cur->start/90/1000%60,
1015                             (cur->start - sub->start)/90 );
1016
1017                         sub->new_chap |= 0x01;
1018                     }
1019                 #endif
1020
1021                 if( sub->size > 0 )
1022                 {
1023                     if( subtitle->config.dest == RENDERSUB )
1024                     {
1025                         /*
1026                          * Tack onto the video buffer for rendering.
1027                          *
1028                          * Note that there may be multiple subtitles
1029                          * whose time intervals overlap which must display
1030                          * on the same frame.
1031                          */
1032                         hb_buffer_t * old_sublist_head = cur->sub;
1033
1034                         /* FIXME: we should avoid this memcpy */
1035                         cur->sub = copy_subtitle( sub );
1036                         cur->sub->next = old_sublist_head;
1037                         cur->sub->start = sub_start;
1038                         cur->sub->stop = sub_stop;
1039
1040                         // Leave the subtitle on the raw queue
1041                         // (until it no longer needs to be displayed)
1042                     } else {
1043                         /*
1044                          * Pass-Through, pop it off of the raw queue,
1045                          */
1046                         sub = hb_fifo_get( subtitle->fifo_raw );
1047                         sub->start = sub_start;
1048                         sub->stop = sub_stop;
1049                         hb_fifo_push( subtitle->fifo_sync, sub );
1050                     }
1051                 } else {
1052                     /*
1053                     * EOF - consume for rendered, else pass through
1054                     */
1055                     if( subtitle->config.dest == RENDERSUB )
1056                     {
1057                         sub = hb_fifo_get( subtitle->fifo_raw );
1058                         hb_buffer_close( &sub );
1059                     } else {
1060                         sub = hb_fifo_get( subtitle->fifo_raw );
1061                         sub->start = sub_start;
1062                         sub->stop = sub_stop;
1063                         hb_fifo_push( subtitle->fifo_sync, sub );
1064                     }
1065                 }
1066             }
1067         }
1068     } // end subtitles
1069 #else
1070     #error "Must select a subtitle sync algorithm."
1071 #endif
1072
1073     /*
1074      * Adjust the pts of the current frame so that it's contiguous
1075      * with the previous frame. The start time of the current frame
1076      * has to be the end time of the previous frame and the stop
1077      * time has to be the start of the next frame.  We don't
1078      * make any adjustments to the source timestamps other than removing
1079      * the clock offsets (which also removes pts discontinuities).
1080      * This means we automatically encode at the source's frame rate.
1081      * MP2 uses an implicit duration (frames end when the next frame
1082      * starts) but more advanced containers like MP4 use an explicit
1083      * duration. Since we're looking ahead one frame we set the
1084      * explicit stop time from the start time of the next frame.
1085      */
1086     *buf_out = cur;
1087     sync->cur = cur = next;
1088     cur->sub = NULL;
1089     int64_t duration = next_start - start;
1090     sync->pts_skip = 0;
1091     if ( duration <= 0 )
1092     {
1093         hb_log( "sync: invalid video duration %"PRId64", start %"PRId64", next %"PRId64"",
1094                 duration, start, next_start );
1095     }
1096
1097     (*buf_out)->start = sync->next_start;
1098     sync->next_start += duration;
1099     (*buf_out)->stop = sync->next_start;
1100
1101     if ( sync->chap_mark )
1102     {
1103         // we have a pending chapter mark from a recent drop - put it on this
1104         // buffer (this may make it one frame late but we can't do any better).
1105         (*buf_out)->new_chap = sync->chap_mark;
1106         sync->chap_mark = 0;
1107     }
1108
1109     /* Update UI */
1110     UpdateState( w );
1111
1112     return HB_WORK_OK;
1113 }
1114
1115 static hb_buffer_t * copy_subtitle( hb_buffer_t * src_list )
1116 {
1117     hb_buffer_t * dst_list = NULL;
1118
1119     hb_buffer_t * src;
1120     hb_buffer_t * dst;
1121     hb_buffer_t ** dst_ptr = &dst_list;
1122     for ( src = src_list, dst_ptr = &dst_list;
1123           src;
1124           src = src->next_subpicture, dst_ptr = &dst->next_subpicture )
1125     {
1126         (*dst_ptr)  = hb_buffer_init( src->size );
1127         dst         = (*dst_ptr);
1128         dst->x      = src->x;
1129         dst->y      = src->y;
1130         dst->width  = src->width;
1131         dst->height = src->height;
1132         memcpy( dst->data, src->data, src->size );
1133     }
1134
1135     return dst_list;
1136 }
1137
1138 // sync*Init does nothing because sync has a special initializer
1139 // that takes care of initializing video and all audio tracks
1140 int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
1141 {
1142     return 0;
1143 }
1144
1145 hb_work_object_t hb_sync_video =
1146 {
1147     WORK_SYNC_VIDEO,
1148     "Video Synchronization",
1149     syncVideoInit,
1150     syncVideoWork,
1151     syncVideoClose
1152 };
1153
1154 /***********************************************************************
1155  * Close Audio
1156  ***********************************************************************
1157  *
1158  **********************************************************************/
1159 void syncAudioClose( hb_work_object_t * w )
1160 {
1161     hb_work_private_t * pv    = w->private_data;
1162     hb_sync_audio_t   * sync  = &pv->type.audio;
1163
1164     if( w->audio->config.out.codec == HB_ACODEC_AC3_PASS )
1165     {
1166         free( sync->ac3_buf );
1167     }
1168     else
1169     {
1170         src_delete( sync->state );
1171     }
1172
1173     hb_lock( pv->common->mutex );
1174     if ( --pv->common->ref == 0 )
1175     {
1176         hb_unlock( pv->common->mutex );
1177         hb_lock_close( &pv->common->mutex );
1178         free( pv->common );
1179     }
1180     else
1181     {
1182         hb_unlock( pv->common->mutex );
1183     }
1184
1185     free( pv );
1186     w->private_data = NULL;
1187 }
1188
1189 int syncAudioInit( hb_work_object_t * w, hb_job_t * job)
1190 {
1191     return 0;
1192 }
1193
1194 /***********************************************************************
1195  * SyncAudio
1196  ***********************************************************************
1197  *
1198  **********************************************************************/
1199 static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
1200                        hb_buffer_t ** buf_out )
1201 {
1202     hb_work_private_t * pv = w->private_data;
1203     hb_job_t        * job = pv->job;
1204     hb_sync_audio_t * sync = &pv->type.audio;
1205     hb_buffer_t     * buf;
1206     int64_t start;
1207
1208     *buf_out = NULL;
1209     buf = *buf_in;
1210     *buf_in = NULL;
1211     /* if the next buffer is an eof send it downstream */
1212     if ( buf->size <= 0 )
1213     {
1214         hb_buffer_close( &buf );
1215         *buf_out = hb_buffer_init( 0 );
1216         pv->common->first_pts[sync->index+1] = INT64_MAX - 1;
1217         return HB_WORK_DONE;
1218     }
1219
1220     /* Wait till we can determine the initial pts of all streams */
1221     if( pv->common->pts_offset == INT64_MIN )
1222     {
1223         pv->common->first_pts[sync->index+1] = buf->start;
1224         hb_lock( pv->common->mutex );
1225         while( pv->common->pts_offset == INT64_MIN )
1226         {
1227             // Full fifos will make us wait forever, so get the
1228             // pts offset from the available streams if full
1229             if (hb_fifo_is_full(w->fifo_in))
1230             {
1231                 getPtsOffset( w );
1232                 hb_cond_broadcast( pv->common->next_frame );
1233             }
1234             else if ( checkPtsOffset( w ) )
1235                 hb_cond_broadcast( pv->common->next_frame );
1236             else
1237                 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
1238         }
1239         hb_unlock( pv->common->mutex );
1240     }
1241
1242     /* Wait for start frame if doing point-to-point */
1243     hb_lock( pv->common->mutex );
1244     start = buf->start - pv->common->audio_pts_slip;
1245     while ( !pv->common->start_found )
1246     {
1247         if ( pv->common->audio_pts_thresh < 0 )
1248         {
1249             // I would initialize this in hb_sync_init, but
1250             // job->pts_to_start can be modified by reader
1251             // after hb_sync_init is called.
1252             pv->common->audio_pts_thresh = job->pts_to_start;
1253         }
1254         if ( buf->start < pv->common->audio_pts_thresh )
1255         {
1256             hb_buffer_close( &buf );
1257             hb_unlock( pv->common->mutex );
1258             return HB_WORK_OK;
1259         }
1260         while ( !pv->common->start_found &&
1261                 buf->start >= pv->common->audio_pts_thresh )
1262         {
1263             hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
1264         }
1265         start = buf->start - pv->common->audio_pts_slip;
1266     }
1267     if ( start < 0 )
1268     {
1269         hb_buffer_close( &buf );
1270         hb_unlock( pv->common->mutex );
1271         return HB_WORK_OK;
1272     }
1273     hb_unlock( pv->common->mutex );
1274
1275     if( job->frame_to_stop && pv->common->count_frames >= job->frame_to_stop )
1276     {
1277         hb_buffer_close( &buf );
1278         *buf_out = hb_buffer_init( 0 );
1279         return HB_WORK_DONE;
1280     }
1281
1282     if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
1283     {
1284         hb_buffer_close( &buf );
1285         *buf_out = hb_buffer_init( 0 );
1286         return HB_WORK_DONE;
1287     }
1288
1289     if ( start - sync->next_start < 0 )
1290     {
1291         // audio time went backwards.
1292         // If our output clock is more than a half frame ahead of the
1293         // input clock drop this frame to move closer to sync.
1294         // Otherwise drop frames until the input clock matches the output clock.
1295         if ( sync->first_drop || sync->next_start - start > 90*15 )
1296         {
1297             // Discard data that's in the past.
1298             if ( sync->first_drop == 0 )
1299             {
1300                 sync->first_drop = sync->next_start;
1301             }
1302             ++sync->drop_count;
1303             hb_buffer_close( &buf );
1304             return HB_WORK_OK;
1305         }
1306     }
1307     if ( sync->first_drop )
1308     {
1309         // we were dropping old data but input buf time is now current
1310         hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
1311                 "(next %"PRId64", current %"PRId64")", w->audio->id,
1312                 (int)( sync->next_start - sync->first_drop ) / 90,
1313                 sync->drop_count, sync->first_drop, (int64_t)sync->next_start );
1314         sync->first_drop = 0;
1315         sync->drop_count = 0;
1316     }
1317     if ( start - sync->next_start >= (90 * 70) )
1318     {
1319         if ( start - sync->next_start > (90000LL * 60) )
1320         {
1321             // there's a gap of more than a minute between the last
1322             // frame and this. assume we got a corrupted timestamp
1323             // and just drop the next buf.
1324             hb_log( "sync: %d minute time gap in audio %d - dropping buf"
1325                     "  start %"PRId64", next %"PRId64,
1326                     (int)((start - sync->next_start) / (90000*60)),
1327                     w->audio->id, start, (int64_t)sync->next_start );
1328             hb_buffer_close( &buf );
1329             return HB_WORK_OK;
1330         }
1331         /*
1332          * there's a gap of at least 70ms between the last
1333          * frame we processed & the next. Fill it with silence.
1334          * Or in the case of DCA, skip some frames from the
1335          * other streams.
1336          */
1337         if( w->audio->config.out.codec == HB_ACODEC_DCA_PASS )
1338         {
1339             hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
1340                     "  start %"PRId64", next %"PRId64,
1341                     (int)((start - sync->next_start) / 90),
1342                     w->audio->id, start, (int64_t)sync->next_start );
1343             hb_lock( pv->common->mutex );
1344             pv->common->audio_pts_slip += (start - sync->next_start);
1345             pv->common->video_pts_slip += (start - sync->next_start);
1346             hb_unlock( pv->common->mutex );
1347             *buf_out = buf;
1348             return HB_WORK_OK;
1349         }
1350         hb_log( "sync: adding %d ms of silence to audio %d"
1351                 "  start %"PRId64", next %"PRId64,
1352                 (int)((start - sync->next_start) / 90),
1353                 w->audio->id, start, (int64_t)sync->next_start );
1354         InsertSilence( w, start - sync->next_start );
1355     }
1356
1357     /*
1358      * When we get here we've taken care of all the dups and gaps in the
1359      * audio stream and are ready to inject the next input frame into
1360      * the output stream.
1361      */
1362     *buf_out = OutputAudioFrame( w->audio, buf, sync );
1363     return HB_WORK_OK;
1364 }
1365
1366 hb_work_object_t hb_sync_audio =
1367 {
1368     WORK_SYNC_AUDIO,
1369     "AudioSynchronization",
1370     syncAudioInit,
1371     syncAudioWork,
1372     syncAudioClose
1373 };
1374
1375 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i )
1376 {
1377     hb_work_object_t  * w;
1378     hb_work_private_t * pv;
1379     hb_title_t        * title = job->title;
1380     hb_sync_audio_t   * sync;
1381
1382     pv = calloc( 1, sizeof( hb_work_private_t ) );
1383     sync = &pv->type.audio;
1384     sync->index = i;
1385     pv->job    = job;
1386     pv->common = common;
1387     pv->common->ref++;
1388     pv->common->pts_count++;
1389
1390     w = hb_get_work( WORK_SYNC_AUDIO );
1391     w->private_data = pv;
1392     w->audio = hb_list_item( title->list_audio, i );
1393     w->fifo_in = w->audio->priv.fifo_raw;
1394
1395     if( w->audio->config.out.codec == HB_ACODEC_AC3_PASS ||
1396         w->audio->config.out.codec == HB_ACODEC_DCA_PASS )
1397     {
1398         w->fifo_out = w->audio->priv.fifo_out;
1399     }
1400     else
1401     {
1402         w->fifo_out = w->audio->priv.fifo_sync;
1403     }
1404
1405     if( w->audio->config.out.codec == HB_ACODEC_AC3_PASS )
1406     {
1407         /* Have a silent AC-3 frame ready in case we have to fill a
1408            gap */
1409         AVCodec        * codec;
1410         AVCodecContext * c;
1411         short          * zeros;
1412
1413         codec = avcodec_find_encoder( CODEC_ID_AC3 );
1414         c     = avcodec_alloc_context();
1415
1416         c->bit_rate    = w->audio->config.in.bitrate;
1417         c->sample_rate = w->audio->config.in.samplerate;
1418         c->channels    = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( w->audio->config.in.channel_layout );
1419         c->sample_fmt  = AV_SAMPLE_FMT_FLT;
1420
1421         if( hb_avcodec_open( c, codec ) < 0 )
1422         {
1423             hb_log( "sync: avcodec_open failed" );
1424             return;
1425         }
1426
1427         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
1428                                  sizeof( float ) * c->channels, 1 );
1429         sync->ac3_size = w->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
1430                              w->audio->config.in.samplerate / 8;
1431         sync->ac3_buf  = malloc( sync->ac3_size );
1432
1433         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
1434                                   zeros ) != sync->ac3_size )
1435         {
1436             hb_log( "sync: avcodec_encode_audio failed" );
1437         }
1438
1439         free( zeros );
1440         hb_avcodec_close( c );
1441         av_free( c );
1442     }
1443     else
1444     {
1445         /* Initialize libsamplerate */
1446         int error;
1447         sync->state = src_new( SRC_SINC_MEDIUM_QUALITY,
1448             HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1449                 w->audio->config.out.mixdown), &error );
1450         sync->data.end_of_input = 0;
1451     }
1452     hb_list_add( job->list_work, w );
1453 }
1454
1455 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
1456                                        hb_sync_audio_t *sync )
1457 {
1458     int64_t start = (int64_t)sync->next_start;
1459     double duration = buf->stop - buf->start;
1460
1461     if( audio->config.in.samplerate == audio->config.out.samplerate ||
1462         audio->config.out.codec == HB_ACODEC_AC3_PASS ||
1463         audio->config.out.codec == HB_ACODEC_DCA_PASS )
1464     {
1465         /*
1466          * If we don't have to do sample rate conversion or this audio is
1467          * pass-thru just send the input buffer downstream after adjusting
1468          * its timestamps to make the output stream continuous.
1469          */
1470     }
1471     else
1472     {
1473         /* Not pass-thru - do sample rate conversion */
1474         int count_in, count_out;
1475         hb_buffer_t * buf_raw = buf;
1476         int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
1477                             sizeof( float );
1478
1479         count_in  = buf_raw->size / channel_count;
1480         /*
1481          * When using stupid rates like 44.1 there will always be some
1482          * truncation error. E.g., a 1536 sample AC3 frame will turn into a
1483          * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
1484          * the error will build up over time and eventually the audio will
1485          * substantially lag the video. libsamplerate will keep track of the
1486          * fractional sample & give it to us when appropriate if we give it
1487          * an extra sample of space in the output buffer.
1488          */
1489         count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
1490
1491         sync->data.input_frames = count_in;
1492         sync->data.output_frames = count_out;
1493         sync->data.src_ratio = (double)audio->config.out.samplerate /
1494                                (double)audio->config.in.samplerate;
1495
1496         buf = hb_buffer_init( count_out * channel_count );
1497         sync->data.data_in  = (float *) buf_raw->data;
1498         sync->data.data_out = (float *) buf->data;
1499         if( src_process( sync->state, &sync->data ) )
1500         {
1501             /* XXX If this happens, we're screwed */
1502             hb_log( "sync: audio %d src_process failed", audio->id );
1503         }
1504         hb_buffer_close( &buf_raw );
1505
1506         buf->size = sync->data.output_frames_gen * channel_count;
1507         duration = (double)( sync->data.output_frames_gen * 90000 ) /
1508                    audio->config.out.samplerate;
1509     }
1510     buf->frametype = HB_FRAME_AUDIO;
1511     buf->start = start;
1512     sync->next_start += duration;
1513     buf->stop  = (int64_t)sync->next_start;
1514     return buf;
1515 }
1516
1517 static void InsertSilence( hb_work_object_t * w, int64_t duration )
1518 {
1519     hb_work_private_t * pv = w->private_data;
1520     hb_sync_audio_t *sync = &pv->type.audio;
1521     hb_buffer_t     *buf;
1522     hb_fifo_t       *fifo;
1523
1524     // to keep pass-thru and regular audio in sync we generate silence in
1525     // AC3 frame-sized units. If the silence duration isn't an integer multiple
1526     // of the AC3 frame duration we will truncate or round up depending on
1527     // which minimizes the timing error.
1528     const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
1529                           w->audio->config.in.samplerate;
1530     int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
1531
1532     while ( --frame_count >= 0 )
1533     {
1534         if( w->audio->config.out.codec == HB_ACODEC_AC3_PASS )
1535         {
1536             buf        = hb_buffer_init( sync->ac3_size );
1537             buf->start = sync->next_start;
1538             buf->stop  = buf->start + frame_dur;
1539             memcpy( buf->data, sync->ac3_buf, buf->size );
1540             fifo = w->audio->priv.fifo_out;
1541         }
1542         else
1543         {
1544             buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
1545                                      HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1546                                          w->audio->config.out.mixdown) );
1547             buf->start = sync->next_start;
1548             buf->stop  = buf->start + frame_dur;
1549             memset( buf->data, 0, buf->size );
1550             fifo = w->audio->priv.fifo_sync;
1551         }
1552         buf = OutputAudioFrame( w->audio, buf, sync );
1553         hb_fifo_push( fifo, buf );
1554     }
1555 }
1556
1557 static void UpdateState( hb_work_object_t * w )
1558 {
1559     hb_work_private_t * pv = w->private_data;
1560     hb_sync_video_t   * sync = &pv->type.video;
1561     hb_state_t state;
1562
1563     if( !pv->common->count_frames )
1564     {
1565         sync->st_first = hb_get_date();
1566         pv->job->st_pause_date = -1;
1567         pv->job->st_paused = 0;
1568     }
1569     pv->common->count_frames++;
1570
1571     if( hb_get_date() > sync->st_dates[3] + 1000 )
1572     {
1573         memmove( &sync->st_dates[0], &sync->st_dates[1],
1574                  3 * sizeof( uint64_t ) );
1575         memmove( &sync->st_counts[0], &sync->st_counts[1],
1576                  3 * sizeof( uint64_t ) );
1577         sync->st_dates[3]  = hb_get_date();
1578         sync->st_counts[3] = pv->common->count_frames;
1579     }
1580
1581 #define p state.param.working
1582     state.state = HB_STATE_WORKING;
1583     p.progress  = (float) pv->common->count_frames / (float) sync->count_frames_max;
1584     if( p.progress > 1.0 )
1585     {
1586         p.progress = 1.0;
1587     }
1588     p.rate_cur   = 1000.0 *
1589         (float) ( sync->st_counts[3] - sync->st_counts[0] ) /
1590         (float) ( sync->st_dates[3] - sync->st_dates[0] );
1591     if( hb_get_date() > sync->st_first + 4000 )
1592     {
1593         int eta;
1594         p.rate_avg = 1000.0 * (float) sync->st_counts[3] /
1595             (float) ( sync->st_dates[3] - sync->st_first - pv->job->st_paused);
1596         eta = (float) ( sync->count_frames_max - sync->st_counts[3] ) /
1597             p.rate_avg;
1598         p.hours   = eta / 3600;
1599         p.minutes = ( eta % 3600 ) / 60;
1600         p.seconds = eta % 60;
1601     }
1602     else
1603     {
1604         p.rate_avg = 0.0;
1605         p.hours    = -1;
1606         p.minutes  = -1;
1607         p.seconds  = -1;
1608     }
1609 #undef p
1610
1611     hb_set_state( pv->job->h, &state );
1612 }
1613
1614 static void UpdateSearchState( hb_work_object_t * w, int64_t start )
1615 {
1616     hb_work_private_t * pv = w->private_data;
1617     hb_sync_video_t   * sync = &pv->type.video;
1618     hb_state_t state;
1619     uint64_t now;
1620     double avg;
1621
1622     now = hb_get_date();
1623     if( !pv->common->count_frames )
1624     {
1625         sync->st_first = now;
1626         pv->job->st_pause_date = -1;
1627         pv->job->st_paused = 0;
1628     }
1629     pv->common->count_frames++;
1630
1631 #define p state.param.working
1632     state.state = HB_STATE_SEARCHING;
1633     if ( pv->job->frame_to_start )
1634         p.progress  = (float) pv->common->count_frames /
1635                       (float) pv->job->frame_to_start;
1636     else if ( pv->job->pts_to_start )
1637         p.progress  = (float) start / (float) pv->job->pts_to_start;
1638     else
1639         p.progress = 0;
1640     if( p.progress > 1.0 )
1641     {
1642         p.progress = 1.0;
1643     }
1644     if (now > sync->st_first)
1645     {
1646         int eta;
1647
1648         if ( pv->job->frame_to_start )
1649         {
1650             avg = 1000.0 * (double)pv->common->count_frames / (now - sync->st_first);
1651             eta = ( pv->job->frame_to_start - pv->common->count_frames ) / avg;
1652         }
1653         else if ( pv->job->pts_to_start )
1654         {
1655             avg = 1000.0 * (double)start / (now - sync->st_first);
1656             eta = ( pv->job->pts_to_start - start ) / avg;
1657         }
1658         p.hours   = eta / 3600;
1659         p.minutes = ( eta % 3600 ) / 60;
1660         p.seconds = eta % 60;
1661     }
1662     else
1663     {
1664         p.rate_avg = 0.0;
1665         p.hours    = -1;
1666         p.minutes  = -1;
1667         p.seconds  = -1;
1668     }
1669 #undef p
1670
1671     hb_set_state( pv->job->h, &state );
1672 }
1673
1674 static void getPtsOffset( hb_work_object_t * w )
1675 {
1676     hb_work_private_t * pv = w->private_data;
1677     int           i ;
1678     int64_t       first_pts = INT64_MAX;
1679
1680     for( i = 0; i < pv->common->pts_count; i++ )
1681     {
1682         if ( pv->common->first_pts[i] < first_pts )
1683             first_pts = pv->common->first_pts[i];
1684     }
1685     pv->common->video_pts_slip = pv->common->audio_pts_slip = pv->common->pts_offset = first_pts;
1686     return;
1687 }
1688
1689 static int checkPtsOffset( hb_work_object_t * w )
1690 {
1691     hb_work_private_t * pv = w->private_data;
1692     int           i ;
1693
1694     for( i = 0; i < pv->common->pts_count; i++ )
1695     {
1696         if ( pv->common->first_pts[i] == INT64_MAX )
1697             return 0;
1698     }
1699     getPtsOffset( w );
1700     return 1;
1701 }