From 484ff7a92298eaeb5e7edc39895b3a26bed704b3 Mon Sep 17 00:00:00 2001 From: Lajos Molnar Date: Thu, 15 Aug 2013 11:37:47 -0700 Subject: [PATCH] Add subtitle support to VideoView. Change-Id: Ibfde491a624272c4f9733098529ad70c6aa93fe0 Signed-off-by: Lajos Molnar Bug: 10326117 --- core/java/android/widget/VideoView.java | 69 +++++++--- media/java/android/media/MediaPlayer.java | 208 +++++++++++++++++++++++++++++- 2 files changed, 256 insertions(+), 21 deletions(-) diff --git a/core/java/android/widget/VideoView.java b/core/java/android/widget/VideoView.java index 0ddc1316771e..f449797d8053 100644 --- a/core/java/android/widget/VideoView.java +++ b/core/java/android/widget/VideoView.java @@ -29,9 +29,12 @@ import android.media.MediaPlayer.OnCompletionListener; import android.media.MediaPlayer.OnErrorListener; import android.media.MediaPlayer.OnInfoListener; import android.media.Metadata; +import android.media.SubtitleController; +import android.media.WebVttRenderer; import android.net.Uri; import android.util.AttributeSet; import android.util.Log; +import android.util.Pair; import android.view.KeyEvent; import android.view.MotionEvent; import android.view.SurfaceHolder; @@ -54,7 +57,8 @@ import java.util.Vector; * it can be used in any layout manager, and provides various display options * such as scaling and tinting. */ -public class VideoView extends SurfaceView implements MediaPlayerControl { +public class VideoView extends SurfaceView + implements MediaPlayerControl, SubtitleController.Anchor { private String TAG = "VideoView"; // settable by the client private Uri mUri; @@ -208,7 +212,7 @@ public class VideoView extends SurfaceView implements MediaPlayerControl { setFocusable(true); setFocusableInTouchMode(true); requestFocus(); - mPendingSubtitleTracks = 0; + mPendingSubtitleTracks = new Vector>(); mCurrentState = STATE_IDLE; mTargetState = STATE_IDLE; } @@ -256,23 +260,19 @@ public class VideoView extends SurfaceView implements MediaPlayerControl { * specify "und" for the language. */ public void addSubtitleSource(InputStream is, MediaFormat format) { - // always signal unsupported message for now - try { - if (is != null) { - is.close(); - } - } catch (IOException e) { - } - if (mMediaPlayer == null) { - ++mPendingSubtitleTracks; + mPendingSubtitleTracks.add(Pair.create(is, format)); } else { - mInfoListener.onInfo( - mMediaPlayer, MediaPlayer.MEDIA_INFO_UNSUPPORTED_SUBTITLE, 0); + try { + mMediaPlayer.addSubtitleSource(is, format); + } catch (IllegalStateException e) { + mInfoListener.onInfo( + mMediaPlayer, MediaPlayer.MEDIA_INFO_UNSUPPORTED_SUBTITLE, 0); + } } } - private int mPendingSubtitleTracks; + private Vector> mPendingSubtitleTracks; public void stopPlayback() { if (mMediaPlayer != null) { @@ -300,6 +300,15 @@ public class VideoView extends SurfaceView implements MediaPlayerControl { release(false); try { mMediaPlayer = new MediaPlayer(); + // TODO: create SubtitleController in MediaPlayer, but we need + // a context for the subtitle renderers + SubtitleController controller = new SubtitleController( + getContext(), + mMediaPlayer.getMediaTimeProvider(), + mMediaPlayer); + controller.registerRenderer(new WebVttRenderer(getContext(), null)); + mMediaPlayer.setSubtitleAnchor(controller, this); + if (mAudioSession != 0) { mMediaPlayer.setAudioSessionId(mAudioSession); } else { @@ -318,9 +327,13 @@ public class VideoView extends SurfaceView implements MediaPlayerControl { mMediaPlayer.setScreenOnWhilePlaying(true); mMediaPlayer.prepareAsync(); - for (int ix = 0; ix < mPendingSubtitleTracks; ix++) { - mInfoListener.onInfo( - mMediaPlayer, MediaPlayer.MEDIA_INFO_UNSUPPORTED_SUBTITLE, 0); + for (Pair pending: mPendingSubtitleTracks) { + try { + mMediaPlayer.addSubtitleSource(pending.first, pending.second); + } catch (IllegalStateException e) { + mInfoListener.onInfo( + mMediaPlayer, MediaPlayer.MEDIA_INFO_UNSUPPORTED_SUBTITLE, 0); + } } // we don't set the target state here either, but preserve the @@ -340,7 +353,7 @@ public class VideoView extends SurfaceView implements MediaPlayerControl { mErrorListener.onError(mMediaPlayer, MediaPlayer.MEDIA_ERROR_UNKNOWN, 0); return; } finally { - mPendingSubtitleTracks = 0; + mPendingSubtitleTracks.clear(); } } @@ -604,7 +617,7 @@ public class VideoView extends SurfaceView implements MediaPlayerControl { mMediaPlayer.reset(); mMediaPlayer.release(); mMediaPlayer = null; - mPendingSubtitleTracks = 0; + mPendingSubtitleTracks.clear(); mCurrentState = STATE_IDLE; if (cleartargetstate) { mTargetState = STATE_IDLE; @@ -874,4 +887,22 @@ public class VideoView extends SurfaceView implements MediaPlayerControl { overlay.layout(left, top, right, bottom); } } + + /** @hide */ + @Override + public void setSubtitleView(View view) { + if (mSubtitleView == view) { + return; + } + + if (mSubtitleView != null) { + removeOverlay(mSubtitleView); + } + mSubtitleView = view; + if (mSubtitleView != null) { + addOverlay(mSubtitleView); + } + } + + private View mSubtitleView; } diff --git a/media/java/android/media/MediaPlayer.java b/media/java/android/media/MediaPlayer.java index bcb1cbd0aad1..d286be417c9e 100644 --- a/media/java/android/media/MediaPlayer.java +++ b/media/java/android/media/MediaPlayer.java @@ -26,11 +26,13 @@ import android.net.Proxy; import android.net.ProxyProperties; import android.net.Uri; import android.os.Handler; +import android.os.HandlerThread; import android.os.Looper; import android.os.Message; import android.os.Parcel; import android.os.Parcelable; import android.os.ParcelFileDescriptor; +import android.os.Process; import android.os.PowerManager; import android.util.Log; import android.view.Surface; @@ -41,14 +43,18 @@ import android.media.AudioManager; import android.media.MediaFormat; import android.media.MediaTimeProvider; import android.media.MediaTimeProvider.OnMediaTimeListener; +import android.media.SubtitleController; import android.media.SubtitleData; import java.io.File; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; +import java.lang.Runnable; import java.net.InetSocketAddress; import java.util.Map; +import java.util.Scanner; import java.util.Set; import java.util.Vector; import java.lang.ref.WeakReference; @@ -520,7 +526,7 @@ import java.lang.ref.WeakReference; * thread by default has a Looper running). * */ -public class MediaPlayer +public class MediaPlayer implements SubtitleController.Listener { /** Constant to retrieve only the new metadata since the last @@ -594,6 +600,9 @@ public class MediaPlayer } mTimeProvider = new TimeProvider(this); + mOutOfBandSubtitleTracks = new Vector(); + mOpenSubtitleSources = new Vector(); + mInbandSubtitleTracks = new SubtitleTrack[0]; /* Native setup requires a weak reference to our object. * It's easier to create it here than in C++. @@ -1356,6 +1365,22 @@ public class MediaPlayer * data source and calling prepare(). */ public void reset() { + mSelectedSubtitleTrackIndex = -1; + synchronized(mOpenSubtitleSources) { + for (final InputStream is: mOpenSubtitleSources) { + try { + is.close(); + } catch (IOException e) { + } + } + mOpenSubtitleSources.clear(); + } + mOutOfBandSubtitleTracks.clear(); + mInbandSubtitleTracks = new SubtitleTrack[0]; + if (mSubtitleController != null) { + mSubtitleController.reset(); + } + stayAwake(false); _reset(); // make sure none of the listeners get called anymore @@ -1575,6 +1600,12 @@ public class MediaPlayer } } + /** @hide */ + TrackInfo(int type, MediaFormat format) { + mTrackType = type; + mFormat = format; + } + /** * {@inheritDoc} */ @@ -1619,6 +1650,19 @@ public class MediaPlayer * @throws IllegalStateException if it is called in an invalid state. */ public TrackInfo[] getTrackInfo() throws IllegalStateException { + TrackInfo trackInfo[] = getInbandTrackInfo(); + // add out-of-band tracks + TrackInfo allTrackInfo[] = new TrackInfo[trackInfo.length + mOutOfBandSubtitleTracks.size()]; + System.arraycopy(trackInfo, 0, allTrackInfo, 0, trackInfo.length); + int i = trackInfo.length; + for (SubtitleTrack track: mOutOfBandSubtitleTracks) { + allTrackInfo[i] = new TrackInfo(TrackInfo.MEDIA_TRACK_TYPE_SUBTITLE, track.getFormat()); + ++i; + } + return allTrackInfo; + } + + private TrackInfo[] getInbandTrackInfo() throws IllegalStateException { Parcel request = Parcel.obtain(); Parcel reply = Parcel.obtain(); try { @@ -1651,6 +1695,143 @@ public class MediaPlayer return false; } + private SubtitleController mSubtitleController; + + /** @hide */ + public void setSubtitleAnchor( + SubtitleController controller, + SubtitleController.Anchor anchor) { + // TODO: create SubtitleController in MediaPlayer + mSubtitleController = controller; + mSubtitleController.setAnchor(anchor); + } + + private SubtitleTrack[] mInbandSubtitleTracks; + private int mSelectedSubtitleTrackIndex = -1; + private Vector mOutOfBandSubtitleTracks; + private Vector mOpenSubtitleSources; + + private OnSubtitleDataListener mSubtitleDataListener = new OnSubtitleDataListener() { + @Override + public void onSubtitleData(MediaPlayer mp, SubtitleData data) { + int index = data.getTrackIndex(); + if (index >= mInbandSubtitleTracks.length) { + return; + } + SubtitleTrack track = mInbandSubtitleTracks[index]; + if (track != null) { + try { + long runID = data.getStartTimeUs() + 1; + // TODO: move conversion into track + track.onData(new String(data.getData(), "UTF-8"), true /* eos */, runID); + track.setRunDiscardTimeMs( + runID, + (data.getStartTimeUs() + data.getDurationUs()) / 1000); + } catch (java.io.UnsupportedEncodingException e) { + Log.w(TAG, "subtitle data for track " + index + " is not UTF-8 encoded: " + e); + } + } + } + }; + + /** @hide */ + @Override + public void onSubtitleTrackSelected(SubtitleTrack track) { + if (mSelectedSubtitleTrackIndex >= 0) { + deselectTrack(mSelectedSubtitleTrackIndex); + } + mSelectedSubtitleTrackIndex = -1; + setOnSubtitleDataListener(null); + for (int i = 0; i < mInbandSubtitleTracks.length; i++) { + if (mInbandSubtitleTracks[i] == track) { + Log.v(TAG, "Selecting subtitle track " + i); + selectTrack(i); + mSelectedSubtitleTrackIndex = i; + setOnSubtitleDataListener(mSubtitleDataListener); + break; + } + } + // no need to select out-of-band tracks + } + + /** @hide */ + public void addSubtitleSource(InputStream is, MediaFormat format) + throws IllegalStateException + { + final InputStream fIs = is; + final MediaFormat fFormat = format; + + // Ensure all input streams are closed. It is also a handy + // way to implement timeouts in the future. + synchronized(mOpenSubtitleSources) { + mOpenSubtitleSources.add(is); + } + + // process each subtitle in its own thread + final HandlerThread thread = new HandlerThread("SubtitleReadThread", + Process.THREAD_PRIORITY_BACKGROUND + Process.THREAD_PRIORITY_MORE_FAVORABLE); + thread.start(); + Handler handler = new Handler(thread.getLooper()); + handler.post(new Runnable() { + private int addTrack() { + if (fIs == null || mSubtitleController == null) { + return MEDIA_INFO_UNSUPPORTED_SUBTITLE; + } + + SubtitleTrack track = mSubtitleController.addTrack(fFormat); + if (track == null) { + return MEDIA_INFO_UNSUPPORTED_SUBTITLE; + } + + // TODO: do the conversion in the subtitle track + Scanner scanner = new Scanner(fIs, "UTF-8"); + String contents = scanner.useDelimiter("\\A").next(); + synchronized(mOpenSubtitleSources) { + mOpenSubtitleSources.remove(fIs); + } + scanner.close(); + mOutOfBandSubtitleTracks.add(track); + track.onData(contents, true /* eos */, ~0 /* runID: keep forever */); + // update default track selection + mSubtitleController.selectDefaultTrack(); + return MEDIA_INFO_EXTERNAL_METADATA_UPDATE; + } + + public void run() { + int res = addTrack(); + if (mEventHandler != null) { + Message m = mEventHandler.obtainMessage(MEDIA_INFO, res, 0, null); + mEventHandler.sendMessage(m); + } + thread.getLooper().quitSafely(); + } + }); + } + + private void scanInternalSubtitleTracks() { + if (mSubtitleController == null) { + Log.e(TAG, "Should have subtitle controller already set"); + return; + } + + TrackInfo[] tracks = getInbandTrackInfo(); + SubtitleTrack[] inbandTracks = new SubtitleTrack[tracks.length]; + for (int i=0; i < tracks.length; i++) { + if (tracks[i].getTrackType() == TrackInfo.MEDIA_TRACK_TYPE_SUBTITLE) { + if (i < mInbandSubtitleTracks.length) { + inbandTracks[i] = mInbandSubtitleTracks[i]; + } else { + MediaFormat format = MediaFormat.createSubtitleFormat( + "text/vtt", tracks[i].getLanguage()); + SubtitleTrack track = mSubtitleController.addTrack(format); + inbandTracks[i] = track; + } + } + } + mInbandSubtitleTracks = inbandTracks; + mSubtitleController.selectDefaultTrack(); + } + /* TODO: Limit the total number of external timed text source to a reasonable number. */ /** @@ -1841,6 +2022,13 @@ public class MediaPlayer private void selectOrDeselectTrack(int index, boolean select) throws IllegalStateException { + // ignore out-of-band tracks + TrackInfo[] trackInfo = getInbandTrackInfo(); + if (index >= trackInfo.length && + index < trackInfo.length + mOutOfBandSubtitleTracks.size()) { + return; + } + Parcel request = Parcel.obtain(); Parcel reply = Parcel.obtain(); try { @@ -1953,6 +2141,7 @@ public class MediaPlayer } switch(msg.what) { case MEDIA_PREPARED: + scanInternalSubtitleTracks(); if (mOnPreparedListener != null) mOnPreparedListener.onPrepared(mMediaPlayer); return; @@ -2008,9 +2197,18 @@ public class MediaPlayer return; case MEDIA_INFO: - if (msg.arg1 != MEDIA_INFO_VIDEO_TRACK_LAGGING) { + switch (msg.arg1) { + case MEDIA_INFO_VIDEO_TRACK_LAGGING: Log.i(TAG, "Info (" + msg.arg1 + "," + msg.arg2 + ")"); + break; + case MEDIA_INFO_METADATA_UPDATE: + scanInternalSubtitleTracks(); + break; + case MEDIA_INFO_EXTERNAL_METADATA_UPDATE: + msg.arg1 = MEDIA_INFO_METADATA_UPDATE; + break; } + if (mOnInfoListener != null) { mOnInfoListener.onInfo(mMediaPlayer, msg.arg1, msg.arg2); } @@ -2409,6 +2607,12 @@ public class MediaPlayer */ public static final int MEDIA_INFO_METADATA_UPDATE = 802; + /** A new set of external-only metadata is available. Used by + * JAVA framework to avoid triggering track scanning. + * @hide + */ + public static final int MEDIA_INFO_EXTERNAL_METADATA_UPDATE = 803; + /** Failed to handle timed text track properly. * @see android.media.MediaPlayer.OnInfoListener * -- 2.11.0