1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.fr/>.
5 It may be used under the terms of the GNU General Public License. */
10 #include "samplerate.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
23 int64_t next_start; /* start time of next output frame */
24 int64_t next_pts; /* start time of next input frame */
25 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
26 int drop_count; /* count of 'time went backwards' drops */
38 struct hb_work_private_s
41 int busy; // bitmask with one bit for each active input
42 // (bit 0 = video; 1 = audio 0, 2 = audio 1, ...
43 // appropriate bit is cleared when input gets
44 // an eof buf. syncWork returns done when all
48 int64_t next_start; /* start time of next output frame */
49 int64_t next_pts; /* start time of next input frame */
50 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
51 int drop_count; /* count of 'time went backwards' drops */
52 int drops; /* frames dropped to make a cbr video stream */
53 int dups; /* frames duplicated to make a cbr video stream */
57 int chap_mark; /* to propagate chapter mark across a drop */
58 hb_buffer_t * cur; /* The next picture to process */
61 hb_sync_audio_t sync_audio[8];
62 int64_t audio_passthru_slip;
65 uint64_t st_counts[4];
70 /***********************************************************************
72 **********************************************************************/
73 static void InitAudio( hb_work_object_t * w, int i );
74 static void SyncVideo( hb_work_object_t * w );
75 static void SyncAudio( hb_work_object_t * w, int i );
76 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
77 static void UpdateState( hb_work_object_t * w );
79 /***********************************************************************
81 ***********************************************************************
82 * Initialize the work object
83 **********************************************************************/
84 int syncInit( hb_work_object_t * w, hb_job_t * job )
86 hb_title_t * title = job->title;
87 hb_chapter_t * chapter;
90 hb_work_private_t * pv;
92 pv = calloc( 1, sizeof( hb_work_private_t ) );
96 pv->pts_offset = INT64_MIN;
98 /* Calculate how many video frames we are expecting */
101 duration = job->pts_to_stop + 90000;
103 else if( job->frame_to_stop )
105 /* Set the duration to a rough estimate */
106 duration = ( job->frame_to_stop / ( job->vrate / job->vrate_base ) ) * 90000;
111 for( i = job->chapter_start; i <= job->chapter_end; i++ )
113 chapter = hb_list_item( title->list_chapter, i - 1 );
114 duration += chapter->duration;
117 /* 1 second safety so we're sure we won't miss anything */
119 pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
121 hb_log( "sync: expecting %d video frames", pv->count_frames_max );
124 /* Initialize libsamplerate for every audio track we have */
125 if ( ! job->indepth_scan )
127 for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
129 pv->busy |= ( 1 << (i + 1) );
137 /***********************************************************************
139 ***********************************************************************
141 **********************************************************************/
142 void syncClose( hb_work_object_t * w )
144 hb_work_private_t * pv = w->private_data;
145 hb_job_t * job = pv->job;
146 hb_title_t * title = job->title;
147 hb_audio_t * audio = NULL;
152 hb_buffer_close( &pv->cur );
155 hb_log( "sync: got %d frames, %d expected",
156 pv->count_frames, pv->count_frames_max );
158 if (pv->drops || pv->dups )
160 hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
163 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
165 audio = hb_list_item( title->list_audio, i );
166 if( audio->config.out.codec == HB_ACODEC_AC3 )
168 free( pv->sync_audio[i].ac3_buf );
172 src_delete( pv->sync_audio[i].state );
177 w->private_data = NULL;
180 /***********************************************************************
182 ***********************************************************************
183 * The root routine of this work abject
185 * The way this works is that we are syncing the audio to the PTS of
186 * the last video that we processed. That's why we skip the audio sync
187 * if we haven't got a valid PTS from the video yet.
189 **********************************************************************/
190 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
191 hb_buffer_t ** unused2 )
193 hb_work_private_t * pv = w->private_data;
199 for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
201 if ( pv->busy & ( 1 << (i + 1) ) )
205 return ( pv->busy? HB_WORK_OK : HB_WORK_DONE );
208 hb_work_object_t hb_sync =
217 static void InitAudio( hb_work_object_t * w, int i )
219 hb_work_private_t * pv = w->private_data;
220 hb_job_t * job = pv->job;
221 hb_title_t * title = job->title;
222 hb_sync_audio_t * sync;
224 sync = &pv->sync_audio[i];
225 sync->audio = hb_list_item( title->list_audio, i );
227 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
229 /* Have a silent AC-3 frame ready in case we have to fill a
235 codec = avcodec_find_encoder( CODEC_ID_AC3 );
236 c = avcodec_alloc_context();
238 c->bit_rate = sync->audio->config.in.bitrate;
239 c->sample_rate = sync->audio->config.in.samplerate;
240 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
242 if( hb_avcodec_open( c, codec ) < 0 )
244 hb_log( "sync: avcodec_open failed" );
248 zeros = calloc( AC3_SAMPLES_PER_FRAME *
249 sizeof( short ) * c->channels, 1 );
250 sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
251 sync->audio->config.in.samplerate / 8;
252 sync->ac3_buf = malloc( sync->ac3_size );
254 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
255 zeros ) != sync->ac3_size )
257 hb_log( "sync: avcodec_encode_audio failed" );
261 hb_avcodec_close( c );
266 /* Initialize libsamplerate */
268 sync->state = src_new( SRC_SINC_MEDIUM_QUALITY, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
269 sync->data.end_of_input = 0;
273 /***********************************************************************
275 ***********************************************************************
277 **********************************************************************/
278 static void SyncVideo( hb_work_object_t * w )
280 hb_work_private_t * pv = w->private_data;
281 hb_buffer_t * cur, * next, * sub = NULL;
282 hb_job_t * job = pv->job;
283 hb_subtitle_t *subtitle;
286 if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
288 /* We haven't even got a frame yet */
294 /* we got an end-of-stream. Feed it downstream & signal that we're done. */
295 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
300 /* At this point we have a frame to process. Let's check
301 1) if we will be able to push into the fifo ahead
302 2) if the next frame is there already, since we need it to
303 compute the duration of the current frame*/
304 while( !hb_fifo_is_full( job->fifo_sync ) &&
305 ( next = hb_fifo_see( job->fifo_raw ) ) )
307 hb_buffer_t * buf_tmp;
309 if( next->size == 0 )
311 /* we got an end-of-stream. Feed it downstream & signal that
312 * we're done. Note that this means we drop the final frame of
313 * video (we don't know its duration). On DVDs the final frame
314 * is often strange and dropping it seems to be a good idea. */
315 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
319 if( pv->pts_offset == INT64_MIN )
321 /* This is our first frame */
323 if ( cur->start != 0 )
326 * The first pts from a dvd should always be zero but
327 * can be non-zero with a transport or program stream since
328 * we're not guaranteed to start on an IDR frame. If we get
329 * a non-zero initial PTS extend its duration so it behaves
330 * as if it started at zero so that our audio timing will
333 hb_log( "sync: first pts is %lld", cur->start );
338 if( cur->new_chap ) {
339 hb_log("sync got new chapter %d", cur->new_chap );
343 * since the first frame is always 0 and the upstream reader code
344 * is taking care of adjusting for pts discontinuities, we just have
345 * to deal with the next frame's start being in the past. This can
346 * happen when the PTS is adjusted after data loss but video frame
347 * reordering causes some frames with the old clock to appear after
348 * the clock change. This creates frames that overlap in time which
349 * looks to us like time going backward. The downstream muxing code
350 * can deal with overlaps of up to a frame time but anything larger
351 * we handle by dropping frames here.
353 if ( (int64_t)( next->start - cur->start ) <= 0 ||
354 (int64_t)( (cur->start - pv->audio_passthru_slip ) - pv->next_pts ) < 0 )
356 if ( pv->first_drop == 0 )
358 pv->first_drop = next->start;
361 buf_tmp = hb_fifo_get( job->fifo_raw );
362 if ( buf_tmp->new_chap )
364 // don't drop a chapter mark when we drop the buffer
365 pv->chap_mark = buf_tmp->new_chap;
367 hb_buffer_close( &buf_tmp );
370 if ( pv->first_drop )
372 hb_log( "sync: video time didn't advance - dropped %d frames "
373 "(delta %d ms, current %lld, next %lld, dur %d)",
374 pv->drop_count, (int)( cur->start - pv->first_drop ) / 90,
375 cur->start, next->start, (int)( next->start - cur->start ) );
381 * Track the video sequence number localy so that we can sync the audio
382 * to it using the sequence number as well as the PTS.
384 pv->video_sequence = cur->sequence;
387 * Look for a subtitle for this frame.
389 * If found then it will be tagged onto a video buffer of the correct time and
390 * sent in to the render pipeline. This only needs to be done for VOBSUBs which
391 * get rendered, other types of subtitles can just sit in their raw_queue until
392 * delt with at muxing.
394 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
396 subtitle = hb_list_item( job->list_subtitle, i );
397 if( subtitle->dest == RENDERSUB )
400 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
402 /* If two subtitles overlap, make the first one stop
403 when the second one starts */
404 sub2 = hb_fifo_see2( subtitle->fifo_raw );
405 if( sub2 && sub->stop > sub2->start )
406 sub->stop = sub2->start;
408 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
409 // sub, cur->sequence, sub->sequence);
411 if( sub->sequence > cur->sequence )
414 * The video is behind where we are, so wait until
415 * it catches up to the same reader point on the
416 * DVD. Then our PTS should be in the same region
423 if( sub->stop > cur->start ) {
425 * The stop time is in the future, so fall through
426 * and we'll deal with it in the next block of
433 * The subtitle is older than this picture, trash it
435 sub = hb_fifo_get( subtitle->fifo_raw );
436 hb_buffer_close( &sub );
440 * There is a valid subtitle, is it time to display it?
444 if( sub->stop > sub->start)
447 * Normal subtitle which ends after it starts, check to
448 * see that the current video is between the start and end.
450 if( cur->start > sub->start &&
451 cur->start < sub->stop )
454 * We should be playing this, so leave the
457 * fall through to display
459 if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
462 * Subtitle is on for less than three seconds, extend
463 * the time that it is displayed to make it easier
464 * to read. Make it 3 seconds or until the next
465 * subtitle is displayed.
467 * This is in response to Indochine which only
468 * displays subs for 1 second - too fast to read.
470 sub->stop = sub->start + ( 3 * 90000 );
472 sub2 = hb_fifo_see2( subtitle->fifo_raw );
474 if( sub2 && sub->stop > sub2->start )
476 sub->stop = sub2->start;
483 * Defer until the play point is within the subtitle
491 * The end of the subtitle is less than the start, this is a
492 * sign of a PTS discontinuity.
494 if( sub->start > cur->start )
497 * we haven't reached the start time yet, or
498 * we have jumped backwards after having
499 * already started this subtitle.
501 if( cur->start < sub->stop )
504 * We have jumped backwards and so should
505 * continue displaying this subtitle.
507 * fall through to display.
513 * Defer until the play point is within the subtitle
519 * Play this subtitle as the start is greater than our
522 * fall through to display/
531 * Don't overwrite the current sub, we'll check the
532 * other subtitle streams on the next video buffer.
534 * It doesn't make much sense having multiple rendered
535 * subtitle tracks anyway.
542 * Adjust the pts of the current frame so that it's contiguous
543 * with the previous frame. The start time of the current frame
544 * has to be the end time of the previous frame and the stop
545 * time has to be the start of the next frame. We don't
546 * make any adjustments to the source timestamps other than removing
547 * the clock offsets (which also removes pts discontinuities).
548 * This means we automatically encode at the source's frame rate.
549 * MP2 uses an implicit duration (frames end when the next frame
550 * starts) but more advanced containers like MP4 use an explicit
551 * duration. Since we're looking ahead one frame we set the
552 * explicit stop time from the start time of the next frame.
555 pv->cur = cur = hb_fifo_get( job->fifo_raw );
556 pv->next_pts = cur->start;
557 int64_t duration = cur->start - buf_tmp->start;
560 hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
561 duration, buf_tmp->start, next->start );
564 buf_tmp->start = pv->next_start;
565 pv->next_start += duration;
566 buf_tmp->stop = pv->next_start;
570 // we have a pending chapter mark from a recent drop - put it on this
571 // buffer (this may make it one frame late but we can't do any better).
572 buf_tmp->new_chap = pv->chap_mark;
576 /* If we have a subtitle for this picture, copy it */
577 /* FIXME: we should avoid this memcpy */
580 buf_tmp->sub = hb_buffer_init( sub->size );
581 buf_tmp->sub->x = sub->x;
582 buf_tmp->sub->y = sub->y;
583 buf_tmp->sub->width = sub->width;
584 buf_tmp->sub->height = sub->height;
585 memcpy( buf_tmp->sub->data, sub->data, sub->size );
588 /* Push the frame to the renderer */
589 hb_fifo_push( job->fifo_sync, buf_tmp );
594 if( job->frame_to_stop && pv->count_frames > job->frame_to_stop )
596 // Drop an empty buffer into our output to ensure that things
597 // get flushed all the way out.
598 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
600 hb_log( "sync: reached %d frames, exiting early (%i busy)",
601 pv->count_frames, pv->busy );
605 /* Make sure we won't get more frames then expected */
606 if( pv->count_frames >= pv->count_frames_max * 2)
608 hb_log( "sync: got too many frames (%d), exiting early",
611 // Drop an empty buffer into our output to ensure that things
612 // get flushed all the way out.
613 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
620 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
621 hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
623 int64_t start = sync->next_start;
624 int64_t duration = buf->stop - buf->start;
626 sync->next_pts += duration;
628 if( audio->config.in.samplerate == audio->config.out.samplerate ||
629 audio->config.out.codec == HB_ACODEC_AC3 ||
630 audio->config.out.codec == HB_ACODEC_DCA )
633 * If we don't have to do sample rate conversion or this audio is
634 * pass-thru just send the input buffer downstream after adjusting
635 * its timestamps to make the output stream continuous.
640 /* Not pass-thru - do sample rate conversion */
641 int count_in, count_out;
642 hb_buffer_t * buf_raw = buf;
643 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
646 count_in = buf_raw->size / channel_count;
648 * When using stupid rates like 44.1 there will always be some
649 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
650 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
651 * the error will build up over time and eventually the audio will
652 * substantially lag the video. libsamplerate will keep track of the
653 * fractional sample & give it to us when appropriate if we give it
654 * an extra sample of space in the output buffer.
656 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
658 sync->data.input_frames = count_in;
659 sync->data.output_frames = count_out;
660 sync->data.src_ratio = (double)audio->config.out.samplerate /
661 (double)audio->config.in.samplerate;
663 buf = hb_buffer_init( count_out * channel_count );
664 sync->data.data_in = (float *) buf_raw->data;
665 sync->data.data_out = (float *) buf->data;
666 if( src_process( sync->state, &sync->data ) )
668 /* XXX If this happens, we're screwed */
669 hb_log( "sync: audio %d src_process failed", i );
671 hb_buffer_close( &buf_raw );
673 buf->size = sync->data.output_frames_gen * channel_count;
674 duration = ( sync->data.output_frames_gen * 90000 ) /
675 audio->config.out.samplerate;
677 buf->frametype = HB_FRAME_AUDIO;
679 buf->stop = start + duration;
680 sync->next_start = start + duration;
681 hb_fifo_push( fifo, buf );
684 /***********************************************************************
686 ***********************************************************************
688 **********************************************************************/
689 static void SyncAudio( hb_work_object_t * w, int i )
691 hb_work_private_t * pv = w->private_data;
692 hb_job_t * job = pv->job;
693 hb_sync_audio_t * sync = &pv->sync_audio[i];
694 hb_audio_t * audio = sync->audio;
699 if( audio->config.out.codec == HB_ACODEC_AC3 ||
700 audio->config.out.codec == HB_ACODEC_DCA )
702 fifo = audio->priv.fifo_out;
706 fifo = audio->priv.fifo_sync;
709 while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
711 start = buf->start - pv->audio_passthru_slip;
712 /* if the next buffer is an eof send it downstream */
713 if ( buf->size <= 0 )
715 buf = hb_fifo_get( audio->priv.fifo_raw );
716 hb_fifo_push( fifo, buf );
717 pv->busy &=~ (1 << (i + 1) );
720 if( job->frame_to_stop && pv->count_frames >= job->frame_to_stop )
722 hb_fifo_push( fifo, hb_buffer_init(0) );
723 pv->busy &=~ (1 << (i + 1) );
726 if ( (int64_t)( start - sync->next_pts ) < 0 )
728 // audio time went backwards.
729 // If our output clock is more than a half frame ahead of the
730 // input clock drop this frame to move closer to sync.
731 // Otherwise drop frames until the input clock matches the output clock.
732 if ( sync->first_drop || sync->next_start - start > 90*15 )
734 // Discard data that's in the past.
735 if ( sync->first_drop == 0 )
737 sync->first_drop = sync->next_pts;
740 buf = hb_fifo_get( audio->priv.fifo_raw );
741 hb_buffer_close( &buf );
744 sync->next_pts = start;
746 if ( sync->first_drop )
748 // we were dropping old data but input buf time is now current
749 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
750 "(next %lld, current %lld)", i,
751 (int)( sync->next_pts - sync->first_drop ) / 90,
752 sync->drop_count, sync->first_drop, sync->next_pts );
753 sync->first_drop = 0;
754 sync->drop_count = 0;
755 sync->next_pts = start;
757 if ( start - sync->next_pts >= (90 * 70) )
759 if ( start - sync->next_pts > (90000LL * 60) )
761 // there's a gap of more than a minute between the last
762 // frame and this. assume we got a corrupted timestamp
763 // and just drop the next buf.
764 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
765 " start %lld, next %lld",
766 (int)((start - sync->next_pts) / (90000*60)),
767 i, start, sync->next_pts );
768 buf = hb_fifo_get( audio->priv.fifo_raw );
769 hb_buffer_close( &buf );
773 * there's a gap of at least 70ms between the last
774 * frame we processed & the next. Fill it with silence.
775 * Or in the case of DCA, skip some frames from the
778 if( sync->audio->config.out.codec == HB_ACODEC_DCA )
780 hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
781 " start %lld, next %lld",
782 (int)((start - sync->next_pts) / 90),
783 i, start, sync->next_pts );
784 pv->audio_passthru_slip += (start - sync->next_pts);
787 hb_log( "sync: adding %d ms of silence to audio %d"
788 " start %lld, next %lld",
789 (int)((start - sync->next_pts) / 90),
790 i, start, sync->next_pts );
791 InsertSilence( w, i, start - sync->next_pts );
796 * When we get here we've taken care of all the dups and gaps in the
797 * audio stream and are ready to inject the next input frame into
800 buf = hb_fifo_get( audio->priv.fifo_raw );
801 OutputAudioFrame( job, audio, buf, sync, fifo, i );
805 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
807 hb_work_private_t * pv = w->private_data;
808 hb_job_t *job = pv->job;
809 hb_sync_audio_t *sync = &pv->sync_audio[i];
813 // to keep pass-thru and regular audio in sync we generate silence in
814 // AC3 frame-sized units. If the silence duration isn't an integer multiple
815 // of the AC3 frame duration we will truncate or round up depending on
816 // which minimizes the timing error.
817 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
818 sync->audio->config.in.samplerate;
819 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
821 while ( --frame_count >= 0 )
823 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
825 buf = hb_buffer_init( sync->ac3_size );
826 buf->start = sync->next_pts;
827 buf->stop = buf->start + frame_dur;
828 memcpy( buf->data, sync->ac3_buf, buf->size );
829 fifo = sync->audio->priv.fifo_out;
833 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
834 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
835 sync->audio->config.out.mixdown) );
836 buf->start = sync->next_pts;
837 buf->stop = buf->start + frame_dur;
838 memset( buf->data, 0, buf->size );
839 fifo = sync->audio->priv.fifo_sync;
841 OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
845 static void UpdateState( hb_work_object_t * w )
847 hb_work_private_t * pv = w->private_data;
850 if( !pv->count_frames )
852 pv->st_first = hb_get_date();
856 if( hb_get_date() > pv->st_dates[3] + 1000 )
858 memmove( &pv->st_dates[0], &pv->st_dates[1],
859 3 * sizeof( uint64_t ) );
860 memmove( &pv->st_counts[0], &pv->st_counts[1],
861 3 * sizeof( uint64_t ) );
862 pv->st_dates[3] = hb_get_date();
863 pv->st_counts[3] = pv->count_frames;
866 #define p state.param.working
867 state.state = HB_STATE_WORKING;
868 p.progress = (float) pv->count_frames / (float) pv->count_frames_max;
869 if( p.progress > 1.0 )
873 p.rate_cur = 1000.0 *
874 (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
875 (float) ( pv->st_dates[3] - pv->st_dates[0] );
876 if( hb_get_date() > pv->st_first + 4000 )
879 p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
880 (float) ( pv->st_dates[3] - pv->st_first );
881 eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
883 p.hours = eta / 3600;
884 p.minutes = ( eta % 3600 ) / 60;
885 p.seconds = eta % 60;
896 hb_set_state( pv->job->h, &state );