From e1957358f11031a554c57d4fb46988dd6044acc1 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 12 Sep 2012 10:19:54 -0700 Subject: [PATCH] Various improvements to the wifi display implementation. Using a MediaPuller now, audio and video on their separate threads. No more flushing the RTP stream on audio tracks, since it wastes too much of a full size UDP packet. Change-Id: I53346b4aea739c3142da13bd179428503a3c98b0 --- media/libmediaplayerservice/RemoteDisplay.cpp | 1 + media/libstagefright/wifi-display/Android.mk | 1 + .../wifi-display/source/Converter.cpp | 2 +- .../wifi-display/source/MediaPuller.cpp | 152 ++++++++++++ .../wifi-display/source/MediaPuller.h | 61 +++++ .../wifi-display/source/PlaybackSession.cpp | 268 ++++++++++++++++----- .../wifi-display/source/PlaybackSession.h | 14 +- .../wifi-display/source/RepeaterSource.cpp | 1 + .../wifi-display/source/Serializer.cpp | 14 +- 9 files changed, 448 insertions(+), 66 deletions(-) create mode 100644 media/libstagefright/wifi-display/source/MediaPuller.cpp create mode 100644 media/libstagefright/wifi-display/source/MediaPuller.h diff --git a/media/libmediaplayerservice/RemoteDisplay.cpp b/media/libmediaplayerservice/RemoteDisplay.cpp index 5542bb5d06..5baa3ad456 100644 --- a/media/libmediaplayerservice/RemoteDisplay.cpp +++ b/media/libmediaplayerservice/RemoteDisplay.cpp @@ -28,6 +28,7 @@ RemoteDisplay::RemoteDisplay( : mLooper(new ALooper), mNetSession(new ANetworkSession), mSource(new WifiDisplaySource(mNetSession, client)) { + mLooper->setName("wfd_looper"); mLooper->registerHandler(mSource); mNetSession->start(); diff --git a/media/libstagefright/wifi-display/Android.mk b/media/libstagefright/wifi-display/Android.mk index 0e59b9e82f..681ba4fb43 100644 --- a/media/libstagefright/wifi-display/Android.mk +++ b/media/libstagefright/wifi-display/Android.mk @@ -10,6 +10,7 @@ LOCAL_SRC_FILES:= \ sink/TunnelRenderer.cpp \ sink/WifiDisplaySink.cpp \ source/Converter.cpp \ + source/MediaPuller.cpp \ source/PlaybackSession.cpp \ source/RepeaterSource.cpp \ source/Serializer.cpp \ diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp index b8b8688e07..0b29df947b 100644 --- a/media/libstagefright/wifi-display/source/Converter.cpp +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -90,7 +90,7 @@ status_t Converter::initEncoder() { if (isAudio) { mOutputFormat->setInt32("bitrate", 64000); // 64 kBit/sec } else { - mOutputFormat->setInt32("bitrate", 5000000); // 5Mbit/sec + mOutputFormat->setInt32("bitrate", 10000000); // 10Mbit/sec mOutputFormat->setInt32("frame-rate", 60); mOutputFormat->setInt32("i-frame-interval", 3); // Iframes every 3 secs } diff --git a/media/libstagefright/wifi-display/source/MediaPuller.cpp b/media/libstagefright/wifi-display/source/MediaPuller.cpp new file mode 100644 index 0000000000..786029a589 --- /dev/null +++ b/media/libstagefright/wifi-display/source/MediaPuller.cpp @@ -0,0 +1,152 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaPuller" +#include + +#include "MediaPuller.h" + +#include +#include +#include +#include +#include +#include + +namespace android { + +MediaPuller::MediaPuller( + const sp &source, const sp ¬ify) + : mSource(source), + mNotify(notify), + mPullGeneration(0) { +} + +MediaPuller::~MediaPuller() { +} + +status_t MediaPuller::postSynchronouslyAndReturnError( + const sp &msg) { + sp response; + status_t err = msg->postAndAwaitResponse(&response); + + if (err != OK) { + return err; + } + + if (!response->findInt32("err", &err)) { + err = OK; + } + + return err; +} + +status_t MediaPuller::start() { + return postSynchronouslyAndReturnError(new AMessage(kWhatStart, id())); +} + +status_t MediaPuller::stop() { + return postSynchronouslyAndReturnError(new AMessage(kWhatStop, id())); +} + +void MediaPuller::onMessageReceived(const sp &msg) { + switch (msg->what()) { + case kWhatStart: + case kWhatStop: + { + status_t err; + + if (msg->what() == kWhatStart) { + err = mSource->start(); + + if (err == OK) { + schedulePull(); + } + } else { + err = mSource->stop(); + ++mPullGeneration; + } + + sp response = new AMessage; + response->setInt32("err", err); + + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + response->postReply(replyID); + break; + } + + case kWhatPull: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mPullGeneration) { + break; + } + + MediaBuffer *mbuf; + status_t err = mSource->read(&mbuf); + + if (err != OK) { + if (err == ERROR_END_OF_STREAM) { + ALOGI("stream ended."); + } else { + ALOGE("error %d reading stream.", err); + } + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatEOS); + notify->post(); + } else { + int64_t timeUs; + CHECK(mbuf->meta_data()->findInt64(kKeyTime, &timeUs)); + + sp accessUnit = new ABuffer(mbuf->range_length()); + + memcpy(accessUnit->data(), + (const uint8_t *)mbuf->data() + mbuf->range_offset(), + mbuf->range_length()); + + accessUnit->meta()->setInt64("timeUs", timeUs); + accessUnit->meta()->setPointer("mediaBuffer", mbuf); + + sp notify = mNotify->dup(); + + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", accessUnit); + notify->post(); + + schedulePull(); + } + break; + } + + default: + TRESPASS(); + } +} + +void MediaPuller::schedulePull() { + sp msg = new AMessage(kWhatPull, id()); + msg->setInt32("generation", mPullGeneration); + msg->post(); +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/MediaPuller.h b/media/libstagefright/wifi-display/source/MediaPuller.h new file mode 100644 index 0000000000..5297501819 --- /dev/null +++ b/media/libstagefright/wifi-display/source/MediaPuller.h @@ -0,0 +1,61 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_PULLER_H_ + +#define MEDIA_PULLER_H_ + +#include + +namespace android { + +struct MediaSource; + +struct MediaPuller : public AHandler { + enum { + kWhatEOS, + kWhatAccessUnit + }; + + MediaPuller(const sp &source, const sp ¬ify); + + status_t start(); + status_t stop(); + +protected: + virtual void onMessageReceived(const sp &msg); + virtual ~MediaPuller(); + +private: + enum { + kWhatStart, + kWhatStop, + kWhatPull, + }; + + sp mSource; + sp mNotify; + int32_t mPullGeneration; + + status_t postSynchronouslyAndReturnError(const sp &msg); + void schedulePull(); + + DISALLOW_EVIL_CONSTRUCTORS(MediaPuller); +}; + +} // namespace android + +#endif // MEDIA_PULLER_H_ diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp index ab164076e3..5d0ddf1fe2 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -21,6 +21,7 @@ #include "PlaybackSession.h" #include "Converter.h" +#include "MediaPuller.h" #include "RepeaterSource.h" #include "Serializer.h" #include "TSPacketizer.h" @@ -45,7 +46,8 @@ #include -#define FAKE_VIDEO 0 +//#define FAKE_VIDEO 1 +#define USE_SERIALIZER 0 namespace android { @@ -53,7 +55,11 @@ static size_t kMaxRTPPacketSize = 1500; static size_t kMaxNumTSPacketsPerRTPPacket = (kMaxRTPPacketSize - 12) / 188; struct WifiDisplaySource::PlaybackSession::Track : public RefBase { - Track(const sp &converter); + Track(const sp &pullLooper, + const sp &codecLooper, + const sp &mediaPuller, + const sp &converter); + Track(const sp &format); sp getFormat(); @@ -63,19 +69,34 @@ struct WifiDisplaySource::PlaybackSession::Track : public RefBase { void setPacketizerTrackIndex(size_t index); + status_t start(); + status_t stop(); + protected: virtual ~Track(); private: + sp mPullLooper; + sp mCodecLooper; + sp mMediaPuller; sp mConverter; sp mFormat; + bool mStarted; ssize_t mPacketizerTrackIndex; DISALLOW_EVIL_CONSTRUCTORS(Track); }; -WifiDisplaySource::PlaybackSession::Track::Track(const sp &converter) - : mConverter(converter), +WifiDisplaySource::PlaybackSession::Track::Track( + const sp &pullLooper, + const sp &codecLooper, + const sp &mediaPuller, + const sp &converter) + : mPullLooper(pullLooper), + mCodecLooper(codecLooper), + mMediaPuller(mediaPuller), + mConverter(converter), + mStarted(false), mPacketizerTrackIndex(-1) { } @@ -85,6 +106,7 @@ WifiDisplaySource::PlaybackSession::Track::Track(const sp &format) } WifiDisplaySource::PlaybackSession::Track::~Track() { + stop(); } sp WifiDisplaySource::PlaybackSession::Track::getFormat() { @@ -108,6 +130,40 @@ void WifiDisplaySource::PlaybackSession::Track::setPacketizerTrackIndex(size_t i mPacketizerTrackIndex = index; } +status_t WifiDisplaySource::PlaybackSession::Track::start() { + if (mStarted) { + return INVALID_OPERATION; + } + + status_t err = OK; + + if (mMediaPuller != NULL) { + err = mMediaPuller->start(); + } + + if (err == OK) { + mStarted = true; + } + + return err; +} + +status_t WifiDisplaySource::PlaybackSession::Track::stop() { + if (!mStarted) { + return INVALID_OPERATION; + } + + status_t err = OK; + + if (mMediaPuller != NULL) { + err = mMediaPuller->stop(); + } + + mStarted = false; + + return err; +} + //////////////////////////////////////////////////////////////////////////////// WifiDisplaySource::PlaybackSession::PlaybackSession( @@ -118,6 +174,7 @@ WifiDisplaySource::PlaybackSession::PlaybackSession( mNotify(notify), mLegacyMode(legacyMode), mLastLifesignUs(), + mVideoTrackIndex(-1), mTSQueue(new ABuffer(12 + kMaxNumTSPacketsPerRTPPacket * 188)), mPrevTimeUs(-1ll), mUseInterleavedTCP(false), @@ -234,11 +291,6 @@ WifiDisplaySource::PlaybackSession::~PlaybackSession() { mTracks.clear(); - if (mCodecLooper != NULL) { - mCodecLooper->stop(); - mCodecLooper.clear(); - } - mPacketizer.clear(); if (mSerializer != NULL) { @@ -248,6 +300,8 @@ WifiDisplaySource::PlaybackSession::~PlaybackSession() { mSerializer.clear(); } + mTracks.clear(); + if (mSerializerLooper != NULL) { mSerializerLooper->stop(); mSerializerLooper.clear(); @@ -290,7 +344,23 @@ status_t WifiDisplaySource::PlaybackSession::play() { scheduleSendSR(); } - return mSerializer->start(); + if (mSerializer != NULL) { + return mSerializer->start(); + } + + for (size_t i = 0; i < mTracks.size(); ++i) { + status_t err = mTracks.editValueAt(i)->start(); + + if (err != OK) { + for (size_t j = 0; j < i; ++j) { + mTracks.editValueAt(j)->stop(); + } + + return err; + } + } + + return OK; } status_t WifiDisplaySource::PlaybackSession::pause() { @@ -483,11 +553,18 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived( offset < packets->size(); offset += 188) { bool lastTSPacket = (offset + 188 >= packets->size()); + // We're only going to flush video, audio packets are + // much more frequent and would waste all that space + // available in a full sized UDP packet. + bool flush = + lastTSPacket + && ((ssize_t)trackIndex == mVideoTrackIndex); + appendTSData( packets->data() + offset, 188, true /* timeDiscontinuity */, - lastTSPacket /* flush */); + flush); } #if LOG_TRANSPORT_STREAM @@ -534,21 +611,33 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived( status_t WifiDisplaySource::PlaybackSession::setupPacketizer() { sp msg = new AMessage(kWhatSerializerNotify, id()); - mSerializerLooper = new ALooper; - mSerializerLooper->start(); + mPacketizer = new TSPacketizer; - mSerializer = new Serializer( #if FAKE_VIDEO - true /* throttled */ + return addFakeSources(); #else - false /* throttled */ -#endif - , msg); - mSerializerLooper->registerHandler(mSerializer); + status_t err = addVideoSource(); - mPacketizer = new TSPacketizer; + if (err != OK) { + return err; + } + + return addAudioSource(); +#endif +} +status_t WifiDisplaySource::PlaybackSession::addFakeSources() { #if FAKE_VIDEO + mSerializerLooper = new ALooper; + mSerializerLooper->setName("serializer_looper"); + mSerializerLooper->start(); + + sp msg = new AMessage(kWhatSerializerNotify, id()); + mSerializer = new Serializer( + true /* throttled */, msg); + + mSerializerLooper->registerHandler(mSerializer); + DataSource::RegisterDefaultSniffers(); sp dataSource = @@ -593,60 +682,128 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer() { mTracks.add(index, new Track(format)); } CHECK(haveAudio || haveVideo); -#else - mCodecLooper = new ALooper; - mCodecLooper->start(); +#endif - sp source = new SurfaceMediaSource(width(), height()); + return OK; +} -#if 0 - ssize_t index = mSerializer->addSource(source); +status_t WifiDisplaySource::PlaybackSession::addSource( + bool isVideo, const sp &source, size_t *numInputBuffers) { +#if USE_SERIALIZER + if (mSerializer == NULL) { + mSerializerLooper = new ALooper; + mSerializerLooper->setName("serializer_looper"); + mSerializerLooper->start(); + + sp msg = new AMessage(kWhatSerializerNotify, id()); + mSerializer = new Serializer( + false /* throttled */, msg); + + mSerializerLooper->registerHandler(mSerializer); + } #else - ssize_t index = mSerializer->addSource( - new RepeaterSource(source, 30.0 /* rateHz */)); + sp pullLooper = new ALooper; + pullLooper->setName("pull_looper"); + + pullLooper->start( + false /* runOnCallingThread */, + false /* canCallJava */, + PRIORITY_DEFAULT); #endif - CHECK_GE(index, 0); + sp codecLooper = new ALooper; + codecLooper->setName("codec_looper"); + + codecLooper->start( + false /* runOnCallingThread */, + false /* canCallJava */, + PRIORITY_DEFAULT); + + size_t trackIndex; + + sp notify; + +#if USE_SERIALIZER + trackIndex = mSerializer->addSource(source); +#else + trackIndex = mTracks.size(); + + notify = new AMessage(kWhatSerializerNotify, id()); + notify->setSize("trackIndex", trackIndex); + sp puller = new MediaPuller(source, notify); + pullLooper->registerHandler(puller); +#endif sp format; status_t err = convertMetaDataToMessage(source->getFormat(), &format); CHECK_EQ(err, (status_t)OK); - format->setInt32("store-metadata-in-buffers", true); + if (isVideo) { + format->setInt32("store-metadata-in-buffers", true); - format->setInt32( - "color-format", OMX_COLOR_FormatAndroidOpaque); + format->setInt32( + "color-format", OMX_COLOR_FormatAndroidOpaque); + } - sp notify = new AMessage(kWhatConverterNotify, id()); - notify->setSize("trackIndex", index); + notify = new AMessage(kWhatConverterNotify, id()); + notify->setSize("trackIndex", trackIndex); sp converter = - new Converter(notify, mCodecLooper, format); + new Converter(notify, codecLooper, format); CHECK_EQ(converter->initCheck(), (status_t)OK); - size_t numInputBuffers = converter->getInputBufferCount(); - ALOGI("numInputBuffers to the encoder is %d", numInputBuffers); - looper()->registerHandler(converter); - mTracks.add(index, new Track(converter)); + if (numInputBuffers != NULL) { + *numInputBuffers = converter->getInputBufferCount(); + } + +#if USE_SERIALIZER + mTracks.add(trackIndex, new Track(NULL, codecLooper, NULL, converter)); +#else + mTracks.add(trackIndex, new Track(pullLooper, codecLooper, puller, converter)); +#endif + + if (isVideo) { + mVideoTrackIndex = trackIndex; + } + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::addVideoSource() { + sp source = new SurfaceMediaSource(width(), height()); - sp sm = defaultServiceManager(); - sp binder = sm->getService(String16("SurfaceFlinger")); - sp service = interface_cast(binder); - CHECK(service != NULL); + sp videoSource = + new RepeaterSource(source, 30.0 /* rateHz */); + + size_t numInputBuffers; + status_t err = addSource(true /* isVideo */, videoSource, &numInputBuffers); + + if (err != OK) { + return err; + } // Add one reference to account for the serializer. // Add another two for unknown reasons. - err = source->setMaxAcquiredBufferCount(numInputBuffers + 2); + err = source->setMaxAcquiredBufferCount(15); // XXX numInputBuffers + 2); CHECK_EQ(err, (status_t)OK); mBufferQueue = source->getBufferQueue(); if (mLegacyMode) { + sp sm = defaultServiceManager(); + sp binder = sm->getService(String16("SurfaceFlinger")); + sp service = interface_cast(binder); + CHECK(service != NULL); + service->connectDisplay(mBufferQueue); } + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::addAudioSource() { sp audioSource = new AudioSource( AUDIO_SOURCE_REMOTE_SUBMIX, 48000 /* sampleRate */, @@ -655,26 +812,11 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer() { if (audioSource->initCheck() == OK) { audioSource->setUseLooperTime(true); - index = mSerializer->addSource(audioSource); - CHECK_GE(index, 0); - - sp audioFormat; - err = convertMetaDataToMessage(audioSource->getFormat(), &audioFormat); - CHECK_EQ(err, (status_t)OK); - - sp audioNotify = new AMessage(kWhatConverterNotify, id()); - audioNotify->setSize("trackIndex", index); - - converter = new Converter(audioNotify, mCodecLooper, audioFormat); - looper()->registerHandler(converter); - - mTracks.add(index, new Track(converter)); - - ALOGI("Successfully instantiated audio source."); - } else { - ALOGW("Unable to instantiate audio source"); + return addSource( + false /* isVideo */, audioSource, NULL /* numInputBuffers */); } -#endif + + ALOGW("Unable to instantiate audio source"); return OK; } diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.h b/media/libstagefright/wifi-display/source/PlaybackSession.h index 5c228f6a9d..528a039fc8 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.h +++ b/media/libstagefright/wifi-display/source/PlaybackSession.h @@ -25,6 +25,8 @@ namespace android { struct ABuffer; struct BufferQueue; struct ISurfaceTexture; +struct MediaPuller; +struct MediaSource; struct Serializer; struct TSPacketizer; @@ -88,10 +90,10 @@ private: sp mSerializerLooper; sp mSerializer; sp mPacketizer; - sp mCodecLooper; sp mBufferQueue; KeyedVector > mTracks; + ssize_t mVideoTrackIndex; sp mTSQueue; int64_t mPrevTimeUs; @@ -134,6 +136,16 @@ private: status_t setupPacketizer(); + status_t addFakeSources(); + + status_t addSource( + bool isVideo, + const sp &source, + size_t *numInputBuffers); + + status_t addVideoSource(); + status_t addAudioSource(); + ssize_t appendTSData( const void *data, size_t size, bool timeDiscontinuity, bool flush); diff --git a/media/libstagefright/wifi-display/source/RepeaterSource.cpp b/media/libstagefright/wifi-display/source/RepeaterSource.cpp index 8af4fdf60e..56e8860809 100644 --- a/media/libstagefright/wifi-display/source/RepeaterSource.cpp +++ b/media/libstagefright/wifi-display/source/RepeaterSource.cpp @@ -38,6 +38,7 @@ status_t RepeaterSource::start(MetaData *params) { mFrameCount = 0; mLooper = new ALooper; + mLooper->setName("repeater_looper"); mLooper->start(); mReflector = new AHandlerReflector(this); diff --git a/media/libstagefright/wifi-display/source/Serializer.cpp b/media/libstagefright/wifi-display/source/Serializer.cpp index 0c54bf7dca..78dc4e6eb0 100644 --- a/media/libstagefright/wifi-display/source/Serializer.cpp +++ b/media/libstagefright/wifi-display/source/Serializer.cpp @@ -47,6 +47,7 @@ protected: private: sp mSource; + AString mMIME; bool mStarted; status_t mFinalResult; @@ -62,6 +63,11 @@ Serializer::Track::Track(const sp &source) mFinalResult(OK), mBuffer(NULL), mBufferTimeUs(-1ll) { + const char *mime; + sp meta = mSource->getFormat(); + CHECK(meta->findCString(kKeyMIMEType, &mime)); + + mMIME = mime; } Serializer::Track::~Track() { @@ -106,7 +112,12 @@ void Serializer::Track::readBufferIfNecessary() { return; } + int64_t nowUs = ALooper::GetNowUs(); mFinalResult = mSource->read(&mBuffer); + int64_t delayUs = ALooper::GetNowUs() - nowUs; + + ALOGV("read on track %s took %lld us, got %d bytes", + mMIME.c_str(), delayUs, mBuffer->range_length()); if (mFinalResult != OK) { ALOGI("read failed w/ err %d", mFinalResult); @@ -275,7 +286,8 @@ status_t Serializer::onStart() { } if (err == OK) { - schedulePoll(); + // XXX add a 5 second delay for the client to get ready. + schedulePoll(5000000ll); } return err; -- 2.11.0