2 * Copyright 2012 Michael Chen <omxcodec@gmail.com>
3 * Copyright 2015 The CyanogenMod Project
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 //#define LOG_NDEBUG 0
19 #define LOG_TAG "SoftFFmpegVideo"
20 #include <utils/Log.h>
22 #include "SoftFFmpegVideo.h"
23 #include "FFmpegComponents.h"
25 #include <media/stagefright/foundation/ADebug.h>
26 #include <media/stagefright/foundation/AUtils.h>
27 #include <media/stagefright/foundation/hexdump.h>
28 #include <media/stagefright/MediaDefs.h>
33 static int decoder_reorder_pts = -1;
37 SoftFFmpegVideo::SoftFFmpegVideo(
39 const char *componentRole,
40 OMX_VIDEO_CODINGTYPE codingType,
41 const OMX_CALLBACKTYPE *callbacks,
43 OMX_COMPONENTTYPE **component,
44 enum AVCodecID codecID)
45 : SoftVideoDecoderOMXComponent(name, componentRole, codingType,
46 NULL, 0, 320, 240, callbacks, appData, component),
47 mCodingType(codingType),
48 mFFmpegAlreadyInited(false),
49 mCodecAlreadyOpened(false),
53 mEOSStatus(INPUT_DATA_AVAILABLE),
54 mExtradataReady(false),
55 mIgnoreExtradata(false),
57 mSignalledError(false) {
59 ALOGD("SoftFFmpegVideo component: %s codingType=%d appData: %p", name, codingType, appData);
63 8192 /* inputBufferSize */,
67 CHECK_EQ(initDecoder(codecID), (status_t)OK);
70 SoftFFmpegVideo::~SoftFFmpegVideo() {
71 ALOGV("~SoftFFmpegVideo");
73 if (mFFmpegAlreadyInited) {
78 void SoftFFmpegVideo::setDefaultCtx(AVCodecContext *avctx, const AVCodec *codec) {
81 avctx->workaround_bugs = 1;
83 if(avctx->lowres > codec->max_lowres){
84 ALOGW("The maximum value for lowres supported by the decoder is %d",
86 avctx->lowres= codec->max_lowres;
89 avctx->skip_frame = AVDISCARD_DEFAULT;
90 avctx->skip_idct = AVDISCARD_DEFAULT;
91 avctx->skip_loop_filter = AVDISCARD_ALL;
92 avctx->error_concealment = 3;
93 avctx->thread_count = 0;
95 if(avctx->lowres) avctx->flags |= CODEC_FLAG_EMU_EDGE;
96 if (fast) avctx->flags2 |= CODEC_FLAG2_FAST;
97 if(codec->capabilities & CODEC_CAP_DR1)
98 avctx->flags |= CODEC_FLAG_EMU_EDGE;
101 status_t SoftFFmpegVideo::initDecoder(enum AVCodecID codecID) {
104 status = initFFmpeg();
108 mFFmpegAlreadyInited = true;
110 mCtx = avcodec_alloc_context3(NULL);
113 ALOGE("avcodec_alloc_context failed.");
117 mCtx->codec_type = AVMEDIA_TYPE_VIDEO;
118 mCtx->codec_id = codecID;
119 mCtx->extradata_size = 0;
120 mCtx->extradata = NULL;
124 void SoftFFmpegVideo::deInitDecoder() {
126 if (avcodec_is_open(mCtx)) {
127 avcodec_flush_buffers(mCtx);
129 if (mCtx->extradata) {
130 av_free(mCtx->extradata);
131 mCtx->extradata = NULL;
132 mCtx->extradata_size = 0;
134 if (mCodecAlreadyOpened) {
144 if (mImgConvertCtx) {
145 sws_freeContext(mImgConvertCtx);
146 mImgConvertCtx = NULL;
150 OMX_ERRORTYPE SoftFFmpegVideo::internalGetParameter(
151 OMX_INDEXTYPE index, OMX_PTR params) {
152 //ALOGV("internalGetParameter index:0x%x", index);
154 case OMX_IndexParamVideoWmv:
156 OMX_VIDEO_PARAM_WMVTYPE *profile =
157 (OMX_VIDEO_PARAM_WMVTYPE *)params;
159 if (profile->nPortIndex != kInputPortIndex) {
160 return OMX_ErrorUndefined;
163 profile->eFormat = OMX_VIDEO_WMVFormatUnused;
165 return OMX_ErrorNone;
168 case OMX_IndexParamVideoRv:
170 OMX_VIDEO_PARAM_RVTYPE *profile =
171 (OMX_VIDEO_PARAM_RVTYPE *)params;
173 if (profile->nPortIndex != kInputPortIndex) {
174 return OMX_ErrorUndefined;
177 profile->eFormat = OMX_VIDEO_RVFormatUnused;
179 return OMX_ErrorNone;
184 if (index != (OMX_INDEXTYPE)OMX_IndexParamVideoFFmpeg) {
185 return SoftVideoDecoderOMXComponent::internalGetParameter(index, params);
188 OMX_VIDEO_PARAM_FFMPEGTYPE *profile =
189 (OMX_VIDEO_PARAM_FFMPEGTYPE *)params;
191 if (profile->nPortIndex != kInputPortIndex) {
192 return OMX_ErrorUndefined;
195 profile->eCodecId = AV_CODEC_ID_NONE;
197 profile->nHeight = 0;
199 return OMX_ErrorNone;
204 OMX_ERRORTYPE SoftFFmpegVideo::isRoleSupported(
205 const OMX_PARAM_COMPONENTROLETYPE *roleParams) {
207 i < sizeof(kVideoComponents) / sizeof(kVideoComponents[0]);
209 if (strncmp((const char *)roleParams->cRole,
210 kVideoComponents[i].mRole, OMX_MAX_STRINGNAME_SIZE - 1) == 0) {
211 return OMX_ErrorNone;
214 ALOGE("unsupported role: %s", (const char *)roleParams->cRole);
215 return OMX_ErrorUndefined;
218 OMX_ERRORTYPE SoftFFmpegVideo::internalSetParameter(
219 OMX_INDEXTYPE index, const OMX_PTR params) {
220 //ALOGV("internalSetParameter index:0x%x", index);
222 case OMX_IndexParamStandardComponentRole:
224 const OMX_PARAM_COMPONENTROLETYPE *roleParams =
225 (const OMX_PARAM_COMPONENTROLETYPE *)params;
226 return isRoleSupported(roleParams);
229 case OMX_IndexParamPortDefinition:
231 OMX_PARAM_PORTDEFINITIONTYPE *newParams =
232 (OMX_PARAM_PORTDEFINITIONTYPE *)params;
233 OMX_VIDEO_PORTDEFINITIONTYPE *video_def = &newParams->format.video;
234 OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(newParams->nPortIndex)->mDef;
236 uint32_t oldWidth = def->format.video.nFrameWidth;
237 uint32_t oldHeight = def->format.video.nFrameHeight;
238 uint32_t newWidth = video_def->nFrameWidth;
239 uint32_t newHeight = video_def->nFrameHeight;
240 if (newWidth != oldWidth || newHeight != oldHeight) {
241 bool outputPort = (newParams->nPortIndex == kOutputPortIndex);
243 ALOGV("OMX_IndexParamPortDefinition (output) width=%d height=%d", newWidth, newHeight);
245 // only update (essentially crop) if size changes
249 updatePortDefinitions(true, true);
250 // reset buffer size based on frame size
251 newParams->nBufferSize = def->nBufferSize;
253 // For input port, we only set nFrameWidth and nFrameHeight. Buffer size
254 // is updated when configuring the output port using the max-frame-size,
255 // though client can still request a larger size.
256 ALOGV("OMX_IndexParamPortDefinition (input) width=%d height=%d", newWidth, newHeight);
257 def->format.video.nFrameWidth = newWidth;
258 def->format.video.nFrameHeight = newHeight;
259 mCtx->width = newWidth;
260 mCtx->height = newHeight;
263 return SoftVideoDecoderOMXComponent::internalSetParameter(index, params);
266 case OMX_IndexParamVideoWmv:
268 OMX_VIDEO_PARAM_WMVTYPE *profile =
269 (OMX_VIDEO_PARAM_WMVTYPE *)params;
271 if (profile->nPortIndex != kInputPortIndex) {
272 return OMX_ErrorUndefined;
275 if (profile->eFormat == OMX_VIDEO_WMVFormat7) {
276 mCtx->codec_id = AV_CODEC_ID_WMV1;
277 } else if (profile->eFormat == OMX_VIDEO_WMVFormat8) {
278 mCtx->codec_id = AV_CODEC_ID_WMV2;
279 } else if (profile->eFormat == OMX_VIDEO_WMVFormat9) {
280 mCtx->codec_id = AV_CODEC_ID_WMV3;
282 mCtx->codec_id = AV_CODEC_ID_VC1;
285 return OMX_ErrorNone;
288 case OMX_IndexParamVideoRv:
290 OMX_VIDEO_PARAM_RVTYPE *profile =
291 (OMX_VIDEO_PARAM_RVTYPE *)params;
293 if (profile->nPortIndex != kInputPortIndex) {
294 return OMX_ErrorUndefined;
297 if (profile->eFormat == OMX_VIDEO_RVFormatG2) {
298 mCtx->codec_id = AV_CODEC_ID_RV20;
299 } else if (profile->eFormat == OMX_VIDEO_RVFormat8) {
300 mCtx->codec_id = AV_CODEC_ID_RV30;
301 } else if (profile->eFormat == OMX_VIDEO_RVFormat9) {
302 mCtx->codec_id = AV_CODEC_ID_RV40;
304 ALOGE("unsupported rv codec: 0x%x", profile->eFormat);
305 return OMX_ErrorUndefined;
308 return OMX_ErrorNone;
313 if (index != (OMX_INDEXTYPE)OMX_IndexParamVideoFFmpeg) {
314 return SoftVideoDecoderOMXComponent::internalSetParameter(index, params);
317 OMX_VIDEO_PARAM_FFMPEGTYPE *profile =
318 (OMX_VIDEO_PARAM_FFMPEGTYPE *)params;
320 if (profile->nPortIndex != kInputPortIndex) {
321 return OMX_ErrorUndefined;
324 mCtx->codec_id = (enum AVCodecID)profile->eCodecId;
325 mCtx->width = profile->nWidth;
326 mCtx->height = profile->nHeight;
328 ALOGD("got OMX_IndexParamVideoFFmpeg, "
329 "eCodecId:%d(%s), width:%u, height:%u",
331 avcodec_get_name(mCtx->codec_id),
335 return OMX_ErrorNone;
340 int32_t SoftFFmpegVideo::handleExtradata() {
341 List<BufferInfo *> &inQueue = getPortQueue(kInputPortIndex);
342 BufferInfo *inInfo = *inQueue.begin();
343 OMX_BUFFERHEADERTYPE *inHeader = inInfo->mHeader;
345 ALOGI("got extradata, ignore: %d, size: %u",
346 mIgnoreExtradata, inHeader->nFilledLen);
347 hexdump(inHeader->pBuffer + inHeader->nOffset, inHeader->nFilledLen);
349 if (mIgnoreExtradata) {
350 ALOGI("got extradata, size: %u, but ignore it", inHeader->nFilledLen);
352 if (!mExtradataReady) {
353 //if (mMode == MODE_H264)
354 //it is possible to receive multiple input buffer with OMX_BUFFERFLAG_CODECCONFIG flag.
355 //for example, H264, the first input buffer is SPS, and another is PPS!
356 int orig_extradata_size = mCtx->extradata_size;
357 mCtx->extradata_size += inHeader->nFilledLen;
358 mCtx->extradata = (uint8_t *)realloc(mCtx->extradata,
359 mCtx->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
360 if (!mCtx->extradata) {
361 ALOGE("ffmpeg video decoder failed to alloc extradata memory.");
365 memcpy(mCtx->extradata + orig_extradata_size,
366 inHeader->pBuffer + inHeader->nOffset,
367 inHeader->nFilledLen);
368 memset(mCtx->extradata + mCtx->extradata_size, 0,
369 FF_INPUT_BUFFER_PADDING_SIZE);
373 inQueue.erase(inQueue.begin());
374 inInfo->mOwnedByUs = false;
375 notifyEmptyBufferDone(inHeader);
380 int32_t SoftFFmpegVideo::openDecoder() {
381 if (mCodecAlreadyOpened) {
385 if (!mExtradataReady) {
386 ALOGI("extradata is ready, size: %d", mCtx->extradata_size);
387 hexdump(mCtx->extradata, mCtx->extradata_size);
388 mExtradataReady = true;
391 //find decoder again as codec_id may have changed
392 mCtx->codec = avcodec_find_decoder(mCtx->codec_id);
394 ALOGE("ffmpeg video decoder failed to find codec");
395 return ERR_CODEC_NOT_FOUND;
398 setDefaultCtx(mCtx, mCtx->codec);
400 ALOGD("begin to open ffmpeg decoder(%s) now",
401 avcodec_get_name(mCtx->codec_id));
403 int err = avcodec_open2(mCtx, mCtx->codec, NULL);
405 ALOGE("ffmpeg video decoder failed to initialize. (%s)", av_err2str(err));
406 return ERR_DECODER_OPEN_FAILED;
408 mCodecAlreadyOpened = true;
410 ALOGD("open ffmpeg video decoder(%s) success",
411 avcodec_get_name(mCtx->codec_id));
413 mFrame = av_frame_alloc();
415 ALOGE("oom for video frame");
422 void SoftFFmpegVideo::initPacket(AVPacket *pkt,
423 OMX_BUFFERHEADERTYPE *inHeader) {
424 memset(pkt, 0, sizeof(AVPacket));
428 pkt->data = (uint8_t *)inHeader->pBuffer + inHeader->nOffset;
429 pkt->size = inHeader->nFilledLen;
430 pkt->pts = inHeader->nTimeStamp;
431 pkt->dts = inHeader->nTimeStamp;
435 pkt->pts = AV_NOPTS_VALUE;
439 if (pkt->pts != AV_NOPTS_VALUE)
441 ALOGV("pkt size:%d, pts:%lld", pkt->size, pkt->pts);
443 ALOGV("pkt size:%d, pts:N/A", pkt->size);
448 int32_t SoftFFmpegVideo::decodeVideo() {
449 int len = 0, err = 0;
451 int32_t ret = ERR_OK;
452 bool is_flush = (mEOSStatus != INPUT_DATA_AVAILABLE);
453 List<BufferInfo *> &inQueue = getPortQueue(kInputPortIndex);
454 BufferInfo *inInfo = NULL;
455 OMX_BUFFERHEADERTYPE *inHeader = NULL;
458 inInfo = *inQueue.begin();
459 CHECK(inInfo != NULL);
460 inHeader = inInfo->mHeader;
464 initPacket(&pkt, inHeader);
466 av_frame_unref(mFrame);
468 err = avcodec_decode_video2(mCtx, mFrame, &gotPic, &pkt);
471 ALOGE("ffmpeg video decoder failed to decode frame. (%d)", err);
472 //don't send error to OMXCodec, skip!
476 ALOGI("ffmpeg video decoder failed to get frame.");
477 //stop sending empty packets if the decoder is finished
478 if (is_flush && mCtx->codec->capabilities & CODEC_CAP_DELAY) {
489 inQueue.erase(inQueue.begin());
490 inInfo->mOwnedByUs = false;
491 notifyEmptyBufferDone(inHeader);
497 int32_t SoftFFmpegVideo::drainOneOutputBuffer() {
498 List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex);
499 BufferInfo *outInfo = *outQueue.begin();
500 OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader;
503 int64_t pts = AV_NOPTS_VALUE;
504 uint8_t *dst = outHeader->pBuffer;
506 uint32_t width = outputBufferWidth();
507 uint32_t height = outputBufferHeight();
509 memset(&pict, 0, sizeof(AVPicture));
511 pict.data[1] = dst + width * height;
512 pict.data[2] = pict.data[1] + (width / 2 * height / 2);
513 pict.linesize[0] = width;
514 pict.linesize[1] = width / 2;
515 pict.linesize[2] = width / 2;
517 ALOGV("drainOneOutputBuffer: frame_width=%d frame_height=%d width=%d height=%d ctx_width=%d ctx_height=%d", mFrame->width, mFrame->height, width, height, mCtx->width, mCtx->height);
519 int sws_flags = SWS_BICUBIC;
520 mImgConvertCtx = sws_getCachedContext(mImgConvertCtx,
521 mFrame->width, mFrame->height, (AVPixelFormat)mFrame->format, width, height,
522 PIX_FMT_YUV420P, sws_flags, NULL, NULL, NULL);
523 if (mImgConvertCtx == NULL) {
524 ALOGE("Cannot initialize the conversion context");
525 return ERR_SWS_FAILED;
527 sws_scale(mImgConvertCtx, mFrame->data, mFrame->linesize,
528 0, height, pict.data, pict.linesize);
530 outHeader->nOffset = 0;
531 outHeader->nFilledLen = (outputBufferWidth() * outputBufferHeight() * 3) / 2;
532 outHeader->nFlags = 0;
533 if (mFrame->key_frame) {
534 outHeader->nFlags |= OMX_BUFFERFLAG_SYNCFRAME;
538 if (decoder_reorder_pts == -1) {
539 pts = av_frame_get_best_effort_timestamp(mFrame);
540 } else if (decoder_reorder_pts) {
541 pts = mFrame->pkt_pts;
543 pts = mFrame->pkt_dts;
546 if (pts == AV_NOPTS_VALUE) {
549 outHeader->nTimeStamp = pts; //FIXME pts is right???
552 ALOGV("mFrame pkt_pts: %lld pkt_dts: %lld used %lld", mFrame->pkt_pts, mFrame->pkt_dts, pts);
555 outQueue.erase(outQueue.begin());
556 outInfo->mOwnedByUs = false;
557 notifyFillBufferDone(outHeader);
562 void SoftFFmpegVideo::drainEOSOutputBuffer() {
563 List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex);
564 BufferInfo *outInfo = *outQueue.begin();
565 CHECK(outInfo != NULL);
566 outQueue.erase(outQueue.begin());
567 OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader;
569 ALOGD("ffmpeg video decoder fill eos outbuf");
571 outHeader->nTimeStamp = 0;
572 outHeader->nFilledLen = 0;
573 outHeader->nFlags = OMX_BUFFERFLAG_EOS;
575 outInfo->mOwnedByUs = false;
576 notifyFillBufferDone(outHeader);
578 mEOSStatus = OUTPUT_FRAMES_FLUSHED;
581 void SoftFFmpegVideo::drainAllOutputBuffers() {
582 List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex);
583 if (!mCodecAlreadyOpened) {
584 drainEOSOutputBuffer();
585 mEOSStatus = OUTPUT_FRAMES_FLUSHED;
589 if(!(mCtx->codec->capabilities & CODEC_CAP_DELAY)) {
590 drainEOSOutputBuffer();
591 mEOSStatus = OUTPUT_FRAMES_FLUSHED;
595 while (!outQueue.empty()) {
596 int32_t err = decodeVideo();
598 notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
599 mSignalledError = true;
601 } else if (err == ERR_FLUSHED) {
602 drainEOSOutputBuffer();
605 CHECK_EQ(err, ERR_OK);
607 if (drainOneOutputBuffer() != ERR_OK) {
608 notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
609 mSignalledError = true;
615 bool SoftFFmpegVideo::handlePortSettingsChange() {
616 CropSettingsMode crop = kCropUnSet;
617 uint32_t width = outputBufferWidth();
618 uint32_t height = outputBufferHeight();
619 if (width != (uint32_t)mCtx->width || height != (uint32_t)mCtx->height) {
621 if (mCropWidth != width || mCropHeight != height) {
625 mCropHeight = height;
630 bool portWillReset = false;
631 SoftVideoDecoderOMXComponent::handlePortSettingsChange(
632 &portWillReset, mCtx->width, mCtx->height, crop);
633 return portWillReset;
636 void SoftFFmpegVideo::onQueueFilled(OMX_U32 portIndex __unused) {
637 if (mSignalledError || mOutputPortSettingsChange != NONE) {
641 if (mEOSStatus == OUTPUT_FRAMES_FLUSHED) {
645 List<BufferInfo *> &inQueue = getPortQueue(kInputPortIndex);
646 List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex);
648 while (((mEOSStatus != INPUT_DATA_AVAILABLE) || !inQueue.empty())
649 && !outQueue.empty()) {
651 if (mEOSStatus == INPUT_EOS_SEEN) {
652 drainAllOutputBuffers();
656 BufferInfo *inInfo = *inQueue.begin();
657 if (inInfo == NULL) {
660 OMX_BUFFERHEADERTYPE *inHeader = inInfo->mHeader;
661 if (inHeader == NULL) {
665 if (inHeader->nFlags & OMX_BUFFERFLAG_CODECCONFIG) {
666 ALOGD("ffmpeg got codecconfig buffer");
667 if (handleExtradata() != ERR_OK) {
668 notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
669 mSignalledError = true;
674 if (inHeader->nFlags & OMX_BUFFERFLAG_EOS) {
675 mEOSStatus = INPUT_EOS_SEEN;
678 if (!mCodecAlreadyOpened) {
679 if (openDecoder() != ERR_OK) {
680 notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
681 mSignalledError = true;
686 int32_t err = decodeVideo();
688 notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
689 mSignalledError = true;
691 } else if (err == ERR_FLUSHED) {
692 drainEOSOutputBuffer();
694 } else if (err == ERR_NO_FRM) {
697 CHECK_EQ(err, ERR_OK);
700 if (handlePortSettingsChange()) {
701 ALOGV("PORT RESET w=%d h=%d", mCtx->width, mCtx->height);
705 if (drainOneOutputBuffer() != ERR_OK) {
706 notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
707 mSignalledError = true;
713 void SoftFFmpegVideo::onPortFlushCompleted(OMX_U32 portIndex) {
714 ALOGV("ffmpeg video decoder flush port(%u)", portIndex);
715 if (portIndex == kInputPortIndex) {
717 //Make sure that the next buffer output does not still
718 //depend on fragments from the last one decoded.
719 avcodec_flush_buffers(mCtx);
721 mEOSStatus = INPUT_DATA_AVAILABLE;
725 void SoftFFmpegVideo::onReset() {
727 SoftVideoDecoderOMXComponent::onReset();
728 mSignalledError = false;
729 mExtradataReady = false;
732 SoftOMXComponent* SoftFFmpegVideo::createSoftOMXComponent(
733 const char *name, const OMX_CALLBACKTYPE *callbacks,
734 OMX_PTR appData, OMX_COMPONENTTYPE **component) {
736 OMX_VIDEO_CODINGTYPE codingType = OMX_VIDEO_CodingAutoDetect;
737 char *componentRole = NULL;
738 enum AVCodecID codecID = AV_CODEC_ID_NONE;
740 for (size_t i = 0; i < kNumVideoComponents; ++i) {
741 if (!strcasecmp(name, kVideoComponents[i].mName)) {
742 componentRole = strdup(kVideoComponents[i].mRole);
743 codingType = kVideoComponents[i].mVideoCodingType;
744 codecID = kVideoComponents[i].mCodecID;
749 if (componentRole == NULL) {
753 return new SoftFFmpegVideo(name, componentRole, codingType,
754 callbacks, appData, component, codecID);
757 } // namespace android