1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.m0k.org/>.
5 It may be used under the terms of the GNU General Public License. */
9 #include "samplerate.h"
10 #include "ffmpeg/avcodec.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
23 int64_t next_start; /* start time of next output frame */
24 int64_t next_pts; /* start time of next input frame */
25 int64_t start_silence; /* if we're inserting silence, the time we started */
26 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
27 int drop_count; /* count of 'time went backwards' drops */
28 int inserting_silence;
40 struct hb_work_private_s
46 hb_subtitle_t * subtitle;
48 int64_t next_start; /* start time of next output frame */
49 int64_t next_pts; /* start time of next input frame */
50 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
51 int drop_count; /* count of 'time went backwards' drops */
55 int chap_mark; /* to propagate chapter mark across a drop */
56 hb_buffer_t * cur; /* The next picture to process */
59 hb_sync_audio_t sync_audio[8];
62 uint64_t st_counts[4];
67 /***********************************************************************
69 **********************************************************************/
70 static void InitAudio( hb_work_object_t * w, int i );
71 static int SyncVideo( hb_work_object_t * w );
72 static void SyncAudio( hb_work_object_t * w, int i );
73 static int NeedSilence( hb_work_object_t * w, hb_audio_t *, int i );
74 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
75 static void UpdateState( hb_work_object_t * w );
77 /***********************************************************************
79 ***********************************************************************
80 * Initialize the work object
81 **********************************************************************/
82 int syncInit( hb_work_object_t * w, hb_job_t * job )
84 hb_title_t * title = job->title;
85 hb_chapter_t * chapter;
88 hb_work_private_t * pv;
90 pv = calloc( 1, sizeof( hb_work_private_t ) );
94 pv->pts_offset = INT64_MIN;
97 /* Calculate how many video frames we are expecting */
99 for( i = job->chapter_start; i <= job->chapter_end; i++ )
101 chapter = hb_list_item( title->list_chapter, i - 1 );
102 duration += chapter->duration;
105 /* 1 second safety so we're sure we won't miss anything */
106 pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
108 hb_log( "sync: expecting %d video frames", pv->count_frames_max );
110 /* Initialize libsamplerate for every audio track we have */
111 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
116 /* Get subtitle info, if any */
117 pv->subtitle = hb_list_item( title->list_subtitle, 0 );
119 pv->video_sequence = 0;
124 /***********************************************************************
126 ***********************************************************************
128 **********************************************************************/
129 void syncClose( hb_work_object_t * w )
131 hb_work_private_t * pv = w->private_data;
132 hb_job_t * job = pv->job;
133 hb_title_t * title = job->title;
134 hb_audio_t * audio = NULL;
138 if( pv->cur ) hb_buffer_close( &pv->cur );
140 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
142 if ( pv->sync_audio[i].start_silence )
144 hb_log( "sync: added %d ms of silence to audio %d",
145 (int)((pv->sync_audio[i].next_pts -
146 pv->sync_audio[i].start_silence) / 90), i );
149 audio = hb_list_item( title->list_audio, i );
150 if( audio->config.out.codec == HB_ACODEC_AC3 )
152 free( pv->sync_audio[i].ac3_buf );
156 src_delete( pv->sync_audio[i].state );
161 w->private_data = NULL;
164 /***********************************************************************
166 ***********************************************************************
167 * The root routine of this work abject
169 * The way this works is that we are syncing the audio to the PTS of
170 * the last video that we processed. That's why we skip the audio sync
171 * if we haven't got a valid PTS from the video yet.
173 **********************************************************************/
174 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
175 hb_buffer_t ** unused2 )
177 hb_work_private_t * pv = w->private_data;
180 /* If we ever got a video frame, handle audio now */
181 if( pv->pts_offset != INT64_MIN )
183 for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
190 return SyncVideo( w );
193 hb_work_object_t hb_sync =
202 static void InitAudio( hb_work_object_t * w, int i )
204 hb_work_private_t * pv = w->private_data;
205 hb_job_t * job = pv->job;
206 hb_title_t * title = job->title;
207 hb_sync_audio_t * sync;
209 sync = &pv->sync_audio[i];
210 sync->audio = hb_list_item( title->list_audio, i );
212 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
214 /* Have a silent AC-3 frame ready in case we have to fill a
220 codec = avcodec_find_encoder( CODEC_ID_AC3 );
221 c = avcodec_alloc_context();
223 c->bit_rate = sync->audio->config.in.bitrate;
224 c->sample_rate = sync->audio->config.in.samplerate;
225 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
227 if( avcodec_open( c, codec ) < 0 )
229 hb_log( "sync: avcodec_open failed" );
233 zeros = calloc( AC3_SAMPLES_PER_FRAME *
234 sizeof( short ) * c->channels, 1 );
235 sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
236 sync->audio->config.in.samplerate / 8;
237 sync->ac3_buf = malloc( sync->ac3_size );
239 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
240 zeros ) != sync->ac3_size )
242 hb_log( "sync: avcodec_encode_audio failed" );
251 /* Initialize libsamplerate */
253 sync->state = src_new( SRC_LINEAR, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
254 sync->data.end_of_input = 0;
258 /***********************************************************************
260 ***********************************************************************
262 **********************************************************************/
263 static int SyncVideo( hb_work_object_t * w )
265 hb_work_private_t * pv = w->private_data;
266 hb_buffer_t * cur, * next, * sub = NULL;
267 hb_job_t * job = pv->job;
274 if( hb_thread_has_exited( job->reader ) &&
275 !hb_fifo_size( job->fifo_mpeg2 ) &&
276 !hb_fifo_size( job->fifo_raw ) )
278 /* All video data has been processed already, we won't get
280 hb_log( "sync: got %d frames, %d expected",
281 pv->count_frames, pv->count_frames_max );
284 hb_buffer_t * buf_tmp;
286 // Drop an empty buffer into our output to ensure that things
287 // get flushed all the way out.
288 buf_tmp = hb_buffer_init(0); // Empty end buffer
289 hb_fifo_push( job->fifo_sync, buf_tmp );
294 if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
296 /* We haven't even got a frame yet */
301 /* At this point we have a frame to process. Let's check
302 1) if we will be able to push into the fifo ahead
303 2) if the next frame is there already, since we need it to
304 compute the duration of the current frame*/
305 while( !hb_fifo_is_full( job->fifo_sync ) &&
306 ( next = hb_fifo_see( job->fifo_raw ) ) )
308 hb_buffer_t * buf_tmp;
310 if( pv->pts_offset == INT64_MIN )
312 /* This is our first frame */
314 if ( cur->start != 0 )
317 * The first pts from a dvd should always be zero but
318 * can be non-zero with a transport or program stream since
319 * we're not guaranteed to start on an IDR frame. If we get
320 * a non-zero initial PTS extend its duration so it behaves
321 * as if it started at zero so that our audio timing will
324 hb_log( "sync: first pts is %lld", cur->start );
330 * since the first frame is always 0 and the upstream reader code
331 * is taking care of adjusting for pts discontinuities, we just have
332 * to deal with the next frame's start being in the past. This can
333 * happen when the PTS is adjusted after data loss but video frame
334 * reordering causes some frames with the old clock to appear after
335 * the clock change. This creates frames that overlap in time which
336 * looks to us like time going backward. The downstream muxing code
337 * can deal with overlaps of up to a frame time but anything larger
338 * we handle by dropping frames here.
340 if ( (int64_t)( next->start - pv->next_pts ) <= 0 )
342 if ( pv->first_drop == 0 )
344 pv->first_drop = next->start;
347 buf_tmp = hb_fifo_get( job->fifo_raw );
348 if ( buf_tmp->new_chap )
350 // don't drop a chapter mark when we drop the buffer
351 pv->chap_mark = buf_tmp->new_chap;
353 hb_buffer_close( &buf_tmp );
356 if ( pv->first_drop )
358 hb_log( "sync: video time didn't advance - dropped %d frames "
359 "(delta %d ms, current %lld, next %lld)",
360 pv->drop_count, (int)( pv->next_pts - pv->first_drop ) / 90,
361 pv->next_pts, pv->first_drop );
367 * Track the video sequence number localy so that we can sync the audio
368 * to it using the sequence number as well as the PTS.
370 pv->video_sequence = cur->sequence;
372 /* Look for a subtitle for this frame */
376 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
378 /* If two subtitles overlap, make the first one stop
379 when the second one starts */
380 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
381 if( sub2 && sub->stop > sub2->start )
382 sub->stop = sub2->start;
384 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
385 // sub, cur->sequence, sub->sequence);
387 if( sub->sequence > cur->sequence )
390 * The video is behind where we are, so wait until
391 * it catches up to the same reader point on the
392 * DVD. Then our PTS should be in the same region
399 if( sub->stop > cur->start ) {
401 * The stop time is in the future, so fall through
402 * and we'll deal with it in the next block of
409 * The subtitle is older than this picture, trash it
411 sub = hb_fifo_get( pv->subtitle->fifo_raw );
412 hb_buffer_close( &sub );
416 * There is a valid subtitle, is it time to display it?
420 if( sub->stop > sub->start)
423 * Normal subtitle which ends after it starts, check to
424 * see that the current video is between the start and end.
426 if( cur->start > sub->start &&
427 cur->start < sub->stop )
430 * We should be playing this, so leave the
433 * fall through to display
435 if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
438 * Subtitle is on for less than three seconds, extend
439 * the time that it is displayed to make it easier
440 * to read. Make it 3 seconds or until the next
441 * subtitle is displayed.
443 * This is in response to Indochine which only
444 * displays subs for 1 second - too fast to read.
446 sub->stop = sub->start + ( 3 * 90000 );
448 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
450 if( sub2 && sub->stop > sub2->start )
452 sub->stop = sub2->start;
459 * Defer until the play point is within the subtitle
467 * The end of the subtitle is less than the start, this is a
468 * sign of a PTS discontinuity.
470 if( sub->start > cur->start )
473 * we haven't reached the start time yet, or
474 * we have jumped backwards after having
475 * already started this subtitle.
477 if( cur->start < sub->stop )
480 * We have jumped backwards and so should
481 * continue displaying this subtitle.
483 * fall through to display.
489 * Defer until the play point is within the subtitle
495 * Play this subtitle as the start is greater than our
498 * fall through to display/
506 * Adjust the pts of the current frame so that it's contiguous
507 * with the previous frame. The start time of the current frame
508 * has to be the end time of the previous frame and the stop
509 * time has to be the start of the next frame. We don't
510 * make any adjustments to the source timestamps other than removing
511 * the clock offsets (which also removes pts discontinuities).
512 * This means we automatically encode at the source's frame rate.
513 * MP2 uses an implicit duration (frames end when the next frame
514 * starts) but more advanced containers like MP4 use an explicit
515 * duration. Since we're looking ahead one frame we set the
516 * explicit stop time from the start time of the next frame.
519 pv->cur = cur = hb_fifo_get( job->fifo_raw );
520 pv->next_pts = next->start;
521 int64_t duration = next->start - buf_tmp->start;
524 hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
525 duration, buf_tmp->start, next->start );
527 buf_tmp->start = pv->next_start;
528 pv->next_start += duration;
529 buf_tmp->stop = pv->next_start;
532 // we have a pending chapter mark from a recent drop - put it on this
533 // buffer (this may make it one frame late but we can't do any better).
534 buf_tmp->new_chap = pv->chap_mark;
538 /* If we have a subtitle for this picture, copy it */
539 /* FIXME: we should avoid this memcpy */
542 buf_tmp->sub = hb_buffer_init( sub->size );
543 buf_tmp->sub->x = sub->x;
544 buf_tmp->sub->y = sub->y;
545 buf_tmp->sub->width = sub->width;
546 buf_tmp->sub->height = sub->height;
547 memcpy( buf_tmp->sub->data, sub->data, sub->size );
550 /* Push the frame to the renderer */
551 hb_fifo_push( job->fifo_sync, buf_tmp );
556 /* Make sure we won't get more frames then expected */
557 if( pv->count_frames >= pv->count_frames_max * 2)
559 hb_log( "sync: got too many frames (%d), exiting early", pv->count_frames );
562 // Drop an empty buffer into our output to ensure that things
563 // get flushed all the way out.
564 buf_tmp = hb_buffer_init(0); // Empty end buffer
565 hb_fifo_push( job->fifo_sync, buf_tmp );
574 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
575 hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
577 int64_t start = sync->next_start;
578 int64_t duration = buf->stop - buf->start;
580 sync->next_pts += duration;
582 if( audio->config.in.samplerate == audio->config.out.samplerate ||
583 audio->config.out.codec == HB_ACODEC_AC3 ||
584 audio->config.out.codec == HB_ACODEC_DCA )
587 * If we don't have to do sample rate conversion or this audio is
588 * pass-thru just send the input buffer downstream after adjusting
589 * its timestamps to make the output stream continuous.
594 /* Not pass-thru - do sample rate conversion */
595 int count_in, count_out;
596 hb_buffer_t * buf_raw = buf;
597 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
600 count_in = buf_raw->size / channel_count;
602 * When using stupid rates like 44.1 there will always be some
603 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
604 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
605 * the error will build up over time and eventually the audio will
606 * substantially lag the video. libsamplerate will keep track of the
607 * fractional sample & give it to us when appropriate if we give it
608 * an extra sample of space in the output buffer.
610 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
612 sync->data.input_frames = count_in;
613 sync->data.output_frames = count_out;
614 sync->data.src_ratio = (double)audio->config.out.samplerate /
615 (double)audio->config.in.samplerate;
617 buf = hb_buffer_init( count_out * channel_count );
618 sync->data.data_in = (float *) buf_raw->data;
619 sync->data.data_out = (float *) buf->data;
620 if( src_process( sync->state, &sync->data ) )
622 /* XXX If this happens, we're screwed */
623 hb_log( "sync: audio %d src_process failed", i );
625 hb_buffer_close( &buf_raw );
627 buf->size = sync->data.output_frames_gen * channel_count;
628 duration = ( sync->data.output_frames_gen * 90000 ) /
629 audio->config.out.samplerate;
631 buf->frametype = HB_FRAME_AUDIO;
633 buf->stop = start + duration;
634 sync->next_start = start + duration;
635 while( hb_fifo_is_full( fifo ) )
638 if ( job->done && hb_fifo_is_full( fifo ) )
640 /* don't block here if the job's finished */
641 hb_buffer_close( &buf );
645 hb_fifo_push( fifo, buf );
648 /***********************************************************************
650 ***********************************************************************
652 **********************************************************************/
653 static void SyncAudio( hb_work_object_t * w, int i )
655 hb_work_private_t * pv = w->private_data;
656 hb_job_t * job = pv->job;
657 hb_sync_audio_t * sync = &pv->sync_audio[i];
658 hb_audio_t * audio = sync->audio;
662 if( audio->config.out.codec == HB_ACODEC_AC3 )
664 fifo = audio->priv.fifo_out;
668 fifo = audio->priv.fifo_sync;
671 while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
673 if ( (int64_t)( buf->start - sync->next_pts ) < 0 )
676 * audio time went backwards by more than a frame time (this can
677 * happen when we reset the PTS because of lost data).
678 * Discard data that's in the past.
680 if ( sync->first_drop == 0 )
682 sync->first_drop = buf->start;
685 buf = hb_fifo_get( audio->priv.fifo_raw );
686 hb_buffer_close( &buf );
689 if ( sync->first_drop )
691 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
692 "(next %lld, current %lld)", i,
693 (int)( sync->next_pts - sync->first_drop ) / 90,
694 sync->drop_count, sync->first_drop, sync->next_pts );
695 sync->first_drop = 0;
696 sync->drop_count = 0;
699 if ( sync->inserting_silence && (int64_t)(buf->start - sync->next_pts) > 0 )
702 * if we're within one frame time of the amount of silence
703 * we need, insert just what we need otherwise insert a frame time.
705 int64_t framedur = buf->stop - buf->start;
706 if ( buf->start - sync->next_pts <= framedur )
708 InsertSilence( w, i, buf->start - sync->next_pts );
709 sync->inserting_silence = 0;
713 InsertSilence( w, i, framedur );
717 if ( buf->start - sync->next_pts >= (90 * 70) )
720 * there's a gap of at least 70ms between the last
721 * frame we processed & the next. Fill it with silence.
723 if ( ! sync->inserting_silence )
725 hb_log( "sync: adding %d ms of silence to audio %d"
726 " start %lld, next %lld",
727 (int)((buf->start - sync->next_pts) / 90),
728 i, buf->start, sync->next_pts );
729 sync->inserting_silence = 1;
731 InsertSilence( w, i, buf->start - sync->next_pts );
736 * When we get here we've taken care of all the dups and gaps in the
737 * audio stream and are ready to inject the next input frame into
740 buf = hb_fifo_get( audio->priv.fifo_raw );
741 OutputAudioFrame( job, audio, buf, sync, fifo, i );
744 if( NeedSilence( w, audio, i ) )
746 InsertSilence( w, i, (90000 * AC3_SAMPLES_PER_FRAME) /
747 sync->audio->config.in.samplerate );
751 static int NeedSilence( hb_work_object_t * w, hb_audio_t * audio, int i )
753 hb_work_private_t * pv = w->private_data;
754 hb_job_t * job = pv->job;
755 hb_sync_audio_t * sync = &pv->sync_audio[i];
757 if( hb_fifo_size( audio->priv.fifo_in ) ||
758 hb_fifo_size( audio->priv.fifo_raw ) ||
759 hb_fifo_size( audio->priv.fifo_sync ) ||
760 hb_fifo_size( audio->priv.fifo_out ) )
762 /* We have some audio, we are fine */
766 /* No audio left in fifos */
768 if( hb_thread_has_exited( job->reader ) )
770 /* We might miss some audio to complete encoding and muxing
772 if ( sync->start_silence == 0 )
774 hb_log("sync: reader has exited, adding silence to audio %d", i);
775 sync->start_silence = sync->next_pts;
782 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
784 hb_work_private_t * pv = w->private_data;
785 hb_job_t *job = pv->job;
786 hb_sync_audio_t *sync = &pv->sync_audio[i];
790 // to keep pass-thru and regular audio in sync we generate silence in
791 // AC3 frame-sized units. If the silence duration isn't an integer multiple
792 // of the AC3 frame duration we will truncate or round up depending on
793 // which minimizes the timing error.
794 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
795 sync->audio->config.in.samplerate;
796 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
798 while ( --frame_count >= 0 )
800 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
802 buf = hb_buffer_init( sync->ac3_size );
803 buf->start = sync->next_pts;
804 buf->stop = buf->start + frame_dur;
805 memcpy( buf->data, sync->ac3_buf, buf->size );
806 fifo = sync->audio->priv.fifo_out;
810 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
811 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
812 sync->audio->config.out.mixdown) );
813 buf->start = sync->next_pts;
814 buf->stop = buf->start + frame_dur;
815 memset( buf->data, 0, buf->size );
816 fifo = sync->audio->priv.fifo_sync;
818 OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
822 static void UpdateState( hb_work_object_t * w )
824 hb_work_private_t * pv = w->private_data;
827 if( !pv->count_frames )
829 pv->st_first = hb_get_date();
833 if( hb_get_date() > pv->st_dates[3] + 1000 )
835 memmove( &pv->st_dates[0], &pv->st_dates[1],
836 3 * sizeof( uint64_t ) );
837 memmove( &pv->st_counts[0], &pv->st_counts[1],
838 3 * sizeof( uint64_t ) );
839 pv->st_dates[3] = hb_get_date();
840 pv->st_counts[3] = pv->count_frames;
843 #define p state.param.working
844 state.state = HB_STATE_WORKING;
845 p.progress = (float) pv->count_frames / (float) pv->count_frames_max;
846 if( p.progress > 1.0 )
850 p.rate_cur = 1000.0 *
851 (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
852 (float) ( pv->st_dates[3] - pv->st_dates[0] );
853 if( hb_get_date() > pv->st_first + 4000 )
856 p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
857 (float) ( pv->st_dates[3] - pv->st_first );
858 eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
860 p.hours = eta / 3600;
861 p.minutes = ( eta % 3600 ) / 60;
862 p.seconds = eta % 60;
873 hb_set_state( pv->job->h, &state );