OSDN Git Service

Allow for streaming of media files (without recompression)
authorAndreas Huber <andih@google.com>
Mon, 18 Mar 2013 18:09:22 +0000 (11:09 -0700)
committerAndreas Huber <andih@google.com>
Mon, 18 Mar 2013 18:09:22 +0000 (11:09 -0700)
Change-Id: I1de356cc37506ba986822d12a1a59e7b64069e02

media/libstagefright/wifi-display/MediaSender.cpp
media/libstagefright/wifi-display/source/PlaybackSession.cpp
media/libstagefright/wifi-display/source/PlaybackSession.h
media/libstagefright/wifi-display/source/TSPacketizer.cpp
media/libstagefright/wifi-display/source/WifiDisplaySource.cpp
media/libstagefright/wifi-display/source/WifiDisplaySource.h
media/libstagefright/wifi-display/wfd.cpp

index e1e957a..a41f81b 100644 (file)
@@ -256,6 +256,37 @@ status_t MediaSender::queueAccessUnit(
                         tsPackets,
                         33 /* packetType */,
                         RTPSender::PACKETIZATION_TRANSPORT_STREAM);
+
+#if 0
+                {
+                    int64_t nowUs = ALooper::GetNowUs();
+
+                    int64_t timeUs;
+                    CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
+
+                    int64_t delayMs = (nowUs - timeUs) / 1000ll;
+
+                    static const int64_t kMinDelayMs = 0;
+                    static const int64_t kMaxDelayMs = 300;
+
+                    const char *kPattern = "########################################";
+                    size_t kPatternSize = strlen(kPattern);
+
+                    int n = (kPatternSize * (delayMs - kMinDelayMs))
+                                / (kMaxDelayMs - kMinDelayMs);
+
+                    if (n < 0) {
+                        n = 0;
+                    } else if ((size_t)n > kPatternSize) {
+                        n = kPatternSize;
+                    }
+
+                    ALOGI("[%lld]: (%4lld ms) %s\n",
+                          timeUs / 1000,
+                          delayMs,
+                          kPattern + kPatternSize - n);
+                }
+#endif
             }
 
             if (err != OK) {
index 94cb2a4..a3b6542 100644 (file)
@@ -39,6 +39,7 @@
 #include <media/stagefright/MediaErrors.h>
 #include <media/stagefright/MediaSource.h>
 #include <media/stagefright/MetaData.h>
+#include <media/stagefright/NuMediaExtractor.h>
 #include <media/stagefright/SurfaceMediaSource.h>
 #include <media/stagefright/Utils.h>
 
@@ -57,6 +58,8 @@ struct WifiDisplaySource::PlaybackSession::Track : public AHandler {
           const sp<MediaPuller> &mediaPuller,
           const sp<Converter> &converter);
 
+    Track(const sp<AMessage> &notify, const sp<AMessage> &format);
+
     void setRepeaterSource(const sp<RepeaterSource> &source);
 
     sp<AMessage> getFormat();
@@ -104,6 +107,7 @@ private:
     sp<ALooper> mCodecLooper;
     sp<MediaPuller> mMediaPuller;
     sp<Converter> mConverter;
+    sp<AMessage> mFormat;
     bool mStarted;
     ssize_t mMediaSenderTrackIndex;
     bool mIsAudio;
@@ -133,6 +137,15 @@ WifiDisplaySource::PlaybackSession::Track::Track(
       mLastOutputBufferQueuedTimeUs(-1ll) {
 }
 
+WifiDisplaySource::PlaybackSession::Track::Track(
+        const sp<AMessage> &notify, const sp<AMessage> &format)
+    : mNotify(notify),
+      mFormat(format),
+      mStarted(false),
+      mIsAudio(IsAudioFormat(format)),
+      mLastOutputBufferQueuedTimeUs(-1ll) {
+}
+
 WifiDisplaySource::PlaybackSession::Track::~Track() {
     CHECK(!mStarted);
 }
@@ -147,7 +160,7 @@ bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat(
 }
 
 sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() {
-    return mConverter->getOutputFormat();
+    return mFormat != NULL ? mFormat : mConverter->getOutputFormat();
 }
 
 bool WifiDisplaySource::PlaybackSession::Track::isAudio() const {
@@ -189,7 +202,9 @@ status_t WifiDisplaySource::PlaybackSession::Track::start() {
 void WifiDisplaySource::PlaybackSession::Track::stopAsync() {
     ALOGV("Track::stopAsync isAudio=%d", mIsAudio);
 
-    mConverter->shutdownAsync();
+    if (mConverter != NULL) {
+        mConverter->shutdownAsync();
+    }
 
     sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, id());
 
@@ -201,6 +216,7 @@ void WifiDisplaySource::PlaybackSession::Track::stopAsync() {
 
         mMediaPuller->stopAsync(msg);
     } else {
+        mStarted = false;
         msg->post();
     }
 }
@@ -324,7 +340,8 @@ WifiDisplaySource::PlaybackSession::PlaybackSession(
         const sp<ANetworkSession> &netSession,
         const sp<AMessage> &notify,
         const in_addr &interfaceAddr,
-        const sp<IHDCP> &hdcp)
+        const sp<IHDCP> &hdcp,
+        const char *path)
     : mNetSession(netSession),
       mNotify(notify),
       mInterfaceAddr(interfaceAddr),
@@ -334,7 +351,14 @@ WifiDisplaySource::PlaybackSession::PlaybackSession(
       mPaused(false),
       mLastLifesignUs(),
       mVideoTrackIndex(-1),
-      mPrevTimeUs(-1ll) {
+      mPrevTimeUs(-1ll),
+      mPullExtractorPending(false),
+      mPullExtractorGeneration(0),
+      mFirstSampleTimeRealUs(-1ll),
+      mFirstSampleTimeUs(-1ll) {
+    if (path != NULL) {
+        mMediaPath.setTo(path);
+    }
 }
 
 status_t WifiDisplaySource::PlaybackSession::init(
@@ -405,10 +429,6 @@ status_t WifiDisplaySource::PlaybackSession::play() {
     return OK;
 }
 
-status_t WifiDisplaySource::PlaybackSession::finishPlay() {
-    return OK;
-}
-
 status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() {
     for (size_t i = 0; i < mTracks.size(); ++i) {
         CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start());
@@ -523,7 +543,10 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived(
                     const sp<Track> &videoTrack =
                         mTracks.valueFor(mVideoTrackIndex);
 
-                    videoTrack->converter()->dropAFrame();
+                    sp<Converter> converter = videoTrack->converter();
+                    if (converter != NULL) {
+                        converter->dropAFrame();
+                    }
                 }
             } else {
                 TRESPASS();
@@ -564,6 +587,12 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived(
 
         case kWhatPause:
         {
+            if (mExtractor != NULL) {
+                ++mPullExtractorGeneration;
+                mFirstSampleTimeRealUs = -1ll;
+                mFirstSampleTimeUs = -1ll;
+            }
+
             if (mPaused) {
                 break;
             }
@@ -578,6 +607,10 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived(
 
         case kWhatResume:
         {
+            if (mExtractor != NULL) {
+                schedulePullExtractor();
+            }
+
             if (!mPaused) {
                 break;
             }
@@ -590,11 +623,152 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived(
             break;
         }
 
+        case kWhatPullExtractorSample:
+        {
+            int32_t generation;
+            CHECK(msg->findInt32("generation", &generation));
+
+            if (generation != mPullExtractorGeneration) {
+                break;
+            }
+
+            mPullExtractorPending = false;
+
+            onPullExtractor();
+            break;
+        }
+
         default:
             TRESPASS();
     }
 }
 
+status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
+        bool enableAudio, bool enableVideo) {
+    DataSource::RegisterDefaultSniffers();
+
+    mExtractor = new NuMediaExtractor;
+
+    status_t err = mExtractor->setDataSource(mMediaPath.c_str());
+
+    if (err != OK) {
+        return err;
+    }
+
+    size_t n = mExtractor->countTracks();
+    bool haveAudio = false;
+    bool haveVideo = false;
+    for (size_t i = 0; i < n; ++i) {
+        sp<AMessage> format;
+        err = mExtractor->getTrackFormat(i, &format);
+
+        if (err != OK) {
+            continue;
+        }
+
+        AString mime;
+        CHECK(format->findString("mime", &mime));
+
+        bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6);
+        bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
+
+        if (isAudio && enableAudio && !haveAudio) {
+            haveAudio = true;
+        } else if (isVideo && enableVideo && !haveVideo) {
+            haveVideo = true;
+        } else {
+            continue;
+        }
+
+        err = mExtractor->selectTrack(i);
+
+        size_t trackIndex = mTracks.size();
+
+        sp<AMessage> notify = new AMessage(kWhatTrackNotify, id());
+        notify->setSize("trackIndex", trackIndex);
+
+        sp<Track> track = new Track(notify, format);
+        looper()->registerHandler(track);
+
+        mTracks.add(trackIndex, track);
+
+        mExtractorTrackToInternalTrack.add(i, trackIndex);
+
+        if (isVideo) {
+            mVideoTrackIndex = trackIndex;
+        }
+
+        uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
+
+        ssize_t mediaSenderTrackIndex =
+            mMediaSender->addTrack(format, flags);
+        CHECK_GE(mediaSenderTrackIndex, 0);
+
+        track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
+
+        if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) {
+            break;
+        }
+    }
+
+    return OK;
+}
+
+void WifiDisplaySource::PlaybackSession::schedulePullExtractor() {
+    if (mPullExtractorPending) {
+        return;
+    }
+
+    int64_t sampleTimeUs;
+    status_t err = mExtractor->getSampleTime(&sampleTimeUs);
+
+    int64_t nowUs = ALooper::GetNowUs();
+
+    if (mFirstSampleTimeRealUs < 0ll) {
+        mFirstSampleTimeRealUs = nowUs;
+        mFirstSampleTimeUs = sampleTimeUs;
+    }
+
+    int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs;
+
+    sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, id());
+    msg->setInt32("generation", mPullExtractorGeneration);
+    msg->post(whenUs - nowUs);
+
+    mPullExtractorPending = true;
+}
+
+void WifiDisplaySource::PlaybackSession::onPullExtractor() {
+    sp<ABuffer> accessUnit = new ABuffer(1024 * 1024);
+    status_t err = mExtractor->readSampleData(accessUnit);
+    if (err != OK) {
+        // EOS.
+        return;
+    }
+
+    int64_t timeUs;
+    CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs));
+
+    accessUnit->meta()->setInt64(
+            "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs);
+
+    size_t trackIndex;
+    CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex));
+
+    sp<AMessage> msg = new AMessage(kWhatConverterNotify, id());
+
+    msg->setSize(
+            "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex));
+
+    msg->setInt32("what", Converter::kWhatAccessUnit);
+    msg->setBuffer("accessUnit", accessUnit);
+    msg->post();
+
+    mExtractor->advance();
+
+    schedulePullExtractor();
+}
+
 status_t WifiDisplaySource::PlaybackSession::setupPacketizer(
         bool enableAudio,
         bool usePCMAudio,
@@ -603,6 +777,10 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer(
         size_t videoResolutionIndex) {
     CHECK(enableAudio || enableVideo);
 
+    if (!mMediaPath.empty()) {
+        return setupMediaPacketizer(enableAudio, enableVideo);
+    }
+
     if (enableVideo) {
         status_t err = addVideoSource(
                 videoResolutionType, videoResolutionIndex);
index cd6da85..da207e2 100644 (file)
@@ -31,6 +31,7 @@ struct IGraphicBufferProducer;
 struct MediaPuller;
 struct MediaSource;
 struct MediaSender;
+struct NuMediaExtractor;
 
 // Encapsulates the state of an RTP/RTCP session in the context of wifi
 // display.
@@ -39,7 +40,8 @@ struct WifiDisplaySource::PlaybackSession : public AHandler {
             const sp<ANetworkSession> &netSession,
             const sp<AMessage> &notify,
             const struct in_addr &interfaceAddr,
-            const sp<IHDCP> &hdcp);
+            const sp<IHDCP> &hdcp,
+            const char *path = NULL);
 
     status_t init(
             const char *clientIP, int32_t clientRtp, int32_t clientRtcp,
@@ -87,12 +89,14 @@ private:
         kWhatPause,
         kWhatResume,
         kWhatMediaSenderNotify,
+        kWhatPullExtractorSample,
     };
 
     sp<ANetworkSession> mNetSession;
     sp<AMessage> mNotify;
     in_addr mInterfaceAddr;
     sp<IHDCP> mHDCP;
+    AString mMediaPath;
 
     sp<MediaSender> mMediaSender;
     int32_t mLocalRTPPort;
@@ -109,6 +113,15 @@ private:
 
     int64_t mPrevTimeUs;
 
+    sp<NuMediaExtractor> mExtractor;
+    KeyedVector<size_t, size_t> mExtractorTrackToInternalTrack;
+    bool mPullExtractorPending;
+    int32_t mPullExtractorGeneration;
+    int64_t mFirstSampleTimeRealUs;
+    int64_t mFirstSampleTimeUs;
+
+    status_t setupMediaPacketizer(bool enableAudio, bool enableVideo);
+
     status_t setupPacketizer(
             bool enableAudio,
             bool usePCMAudio,
@@ -133,6 +146,9 @@ private:
 
     void notifySessionDead();
 
+    void schedulePullExtractor();
+    void onPullExtractor();
+
     DISALLOW_EVIL_CONSTRUCTORS(PlaybackSession);
 };
 
index 53b7187..d993764 100644 (file)
@@ -261,7 +261,7 @@ void TSPacketizer::Track::finalize() {
             data[0] = 40;  // descriptor_tag
             data[1] = 4;  // descriptor_length
 
-            CHECK_EQ(mCSD.size(), 1u);
+            CHECK_GE(mCSD.size(), 1u);
             const sp<ABuffer> &sps = mCSD.itemAt(0);
             CHECK(!memcmp("\x00\x00\x00\x01", sps->data(), 4));
             CHECK_GE(sps->size(), 7u);
index c8798c6..5167cb3 100644 (file)
@@ -44,7 +44,8 @@ namespace android {
 
 WifiDisplaySource::WifiDisplaySource(
         const sp<ANetworkSession> &netSession,
-        const sp<IRemoteDisplayClient> &client)
+        const sp<IRemoteDisplayClient> &client,
+        const char *path)
     : mState(INITIALIZED),
       mNetSession(netSession),
       mClient(client),
@@ -59,7 +60,12 @@ WifiDisplaySource::WifiDisplaySource(
       mIsHDCP2_0(false),
       mHDCPPort(0),
       mHDCPInitializationComplete(false),
-      mSetupTriggerDeferred(false) {
+      mSetupTriggerDeferred(false),
+      mPlaybackSessionEstablished(false) {
+    if (path != NULL) {
+        mMediaPath.setTo(path);
+    }
+
     mSupportedSourceVideoFormats.disableAll();
 
     mSupportedSourceVideoFormats.setNativeResolution(
@@ -389,6 +395,8 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) {
                 mClient->onDisplayError(
                         IRemoteDisplayClient::kDisplayErrorUnknown);
             } else if (what == PlaybackSession::kWhatSessionEstablished) {
+                mPlaybackSessionEstablished = true;
+
                 if (mClient != NULL) {
                     if (!mSinkSupportsVideo) {
                         mClient->onDisplayConnected(
@@ -419,6 +427,8 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) {
                     }
                 }
 
+                finishPlay();
+
                 if (mState == ABOUT_TO_PLAY) {
                     mState = PLAYING;
                 }
@@ -1222,7 +1232,7 @@ status_t WifiDisplaySource::onSetupRequest(
 
     sp<PlaybackSession> playbackSession =
         new PlaybackSession(
-                mNetSession, notify, mInterfaceAddr, mHDCP);
+                mNetSession, notify, mInterfaceAddr, mHDCP, mMediaPath.c_str());
 
     looper()->registerHandler(playbackSession);
 
@@ -1332,16 +1342,18 @@ status_t WifiDisplaySource::onPlayRequest(
     }
 
     ALOGI("Received PLAY request.");
-
-    status_t err = playbackSession->play();
-    CHECK_EQ(err, (status_t)OK);
+    if (mPlaybackSessionEstablished) {
+        finishPlay();
+    } else {
+        ALOGI("deferring PLAY request until session established.");
+    }
 
     AString response = "RTSP/1.0 200 OK\r\n";
     AppendCommonResponse(&response, cseq, playbackSessionID);
     response.append("Range: npt=now-\r\n");
     response.append("\r\n");
 
-    err = mNetSession->sendRequest(sessionID, response.c_str());
+    status_t err = mNetSession->sendRequest(sessionID, response.c_str());
 
     if (err != OK) {
         return err;
@@ -1352,14 +1364,20 @@ status_t WifiDisplaySource::onPlayRequest(
         return OK;
     }
 
-    playbackSession->finishPlay();
-
     CHECK_EQ(mState, AWAITING_CLIENT_PLAY);
     mState = ABOUT_TO_PLAY;
 
     return OK;
 }
 
+void WifiDisplaySource::finishPlay() {
+    const sp<PlaybackSession> &playbackSession =
+        mClientInfo.mPlaybackSession;
+
+    status_t err = playbackSession->play();
+    CHECK_EQ(err, (status_t)OK);
+}
+
 status_t WifiDisplaySource::onPauseRequest(
         int32_t sessionID,
         int32_t cseq,
index 9e72682..3a1b0f9 100644 (file)
@@ -39,7 +39,8 @@ struct WifiDisplaySource : public AHandler {
 
     WifiDisplaySource(
             const sp<ANetworkSession> &netSession,
-            const sp<IRemoteDisplayClient> &client);
+            const sp<IRemoteDisplayClient> &client,
+            const char *path = NULL);
 
     status_t start(const char *iface);
     status_t stop();
@@ -116,6 +117,7 @@ private:
     VideoFormats mSupportedSourceVideoFormats;
     sp<ANetworkSession> mNetSession;
     sp<IRemoteDisplayClient> mClient;
+    AString mMediaPath;
     sp<TimeSyncer> mTimeSyncer;
     struct in_addr mInterfaceAddr;
     int32_t mSessionID;
@@ -161,6 +163,8 @@ private:
     bool mHDCPInitializationComplete;
     bool mSetupTriggerDeferred;
 
+    bool mPlaybackSessionEstablished;
+
     status_t makeHDCP();
     // <<<< HDCP specific section
 
@@ -257,6 +261,8 @@ private:
     void finishStopAfterDisconnectingClient();
     void finishStop2();
 
+    void finishPlay();
+
     DISALLOW_EVIL_CONSTRUCTORS(WifiDisplaySource);
 };
 
index 0b18484..3a7a6e2 100644 (file)
@@ -42,6 +42,7 @@ static void usage(const char *me) {
             "           %s -c host[:port]\tconnect to wifi source\n"
             "               -u uri        \tconnect to an rtsp uri\n"
             "               -l ip[:port] \tlisten on the specified port "
+            "               -f(ilename)  \tstream media "
             "(create a sink)\n",
             me);
 }
@@ -93,22 +94,24 @@ void RemoteDisplayClient::onDisplayConnected(
     ALOGI("onDisplayConnected width=%u, height=%u, flags = 0x%08x",
           width, height, flags);
 
-    mSurfaceTexture = bufferProducer;
-    mDisplayBinder = mComposerClient->createDisplay(
-            String8("foo"), false /* secure */);
+    if (bufferProducer != NULL) {
+        mSurfaceTexture = bufferProducer;
+        mDisplayBinder = mComposerClient->createDisplay(
+                String8("foo"), false /* secure */);
 
-    SurfaceComposerClient::openGlobalTransaction();
-    mComposerClient->setDisplaySurface(mDisplayBinder, mSurfaceTexture);
+        SurfaceComposerClient::openGlobalTransaction();
+        mComposerClient->setDisplaySurface(mDisplayBinder, mSurfaceTexture);
 
-    Rect layerStackRect(1280, 720);  // XXX fix this.
-    Rect displayRect(1280, 720);
+        Rect layerStackRect(1280, 720);  // XXX fix this.
+        Rect displayRect(1280, 720);
 
-    mComposerClient->setDisplayProjection(
-            mDisplayBinder, 0 /* 0 degree rotation */,
-            layerStackRect,
-            displayRect);
+        mComposerClient->setDisplayProjection(
+                mDisplayBinder, 0 /* 0 degree rotation */,
+                layerStackRect,
+                displayRect);
 
-    SurfaceComposerClient::closeGlobalTransaction();
+        SurfaceComposerClient::closeGlobalTransaction();
+    }
 }
 
 void RemoteDisplayClient::onDisplayDisconnected() {
@@ -181,6 +184,24 @@ static void createSource(const AString &addr, int32_t port) {
     enableAudioSubmix(false /* enable */);
 }
 
+static void createFileSource(
+        const AString &addr, int32_t port, const char *path) {
+    sp<ANetworkSession> session = new ANetworkSession;
+    session->start();
+
+    sp<ALooper> looper = new ALooper;
+    looper->start();
+
+    sp<RemoteDisplayClient> client = new RemoteDisplayClient;
+    sp<WifiDisplaySource> source = new WifiDisplaySource(session, client, path);
+    looper->registerHandler(source);
+
+    AString iface = StringPrintf("%s:%d", addr.c_str(), port);
+    CHECK_EQ((status_t)OK, source->start(iface.c_str()));
+
+    client->waitUntilDone();
+}
+
 }  // namespace android
 
 int main(int argc, char **argv) {
@@ -197,8 +218,10 @@ int main(int argc, char **argv) {
     AString listenOnAddr;
     int32_t listenOnPort = -1;
 
+    AString path;
+
     int res;
-    while ((res = getopt(argc, argv, "hc:l:u:")) >= 0) {
+    while ((res = getopt(argc, argv, "hc:l:u:f:")) >= 0) {
         switch (res) {
             case 'c':
             {
@@ -228,6 +251,12 @@ int main(int argc, char **argv) {
                 break;
             }
 
+            case 'f':
+            {
+                path = optarg;
+                break;
+            }
+
             case 'l':
             {
                 const char *colonPos = strrchr(optarg, ':');
@@ -266,7 +295,12 @@ int main(int argc, char **argv) {
     }
 
     if (listenOnPort >= 0) {
-        createSource(listenOnAddr, listenOnPort);
+        if (path.empty()) {
+            createSource(listenOnAddr, listenOnPort);
+        } else {
+            createFileSource(listenOnAddr, listenOnPort, path.c_str());
+        }
+
         exit(0);
     }