// Copyright (C) 2019 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "host-common/MediaH264DecoderFfmpeg.h" #include "aemu/base/system/System.h" #include "host-common/H264NaluParser.h" #include "host-common/H264PingInfoParser.h" #include "host-common/YuvConverter.h" #include #include #include #include #include #define MEDIA_H264_DEBUG 0 #if MEDIA_H264_DEBUG #define H264_DPRINT(fmt,...) fprintf(stderr, "h264-ffmpeg-dec: %s:%d " fmt "\n", __func__, __LINE__, ##__VA_ARGS__); #else #define H264_DPRINT(fmt,...) #endif namespace android { namespace emulation { using InitContextParam = H264PingInfoParser::InitContextParam; using DecodeFrameParam = H264PingInfoParser::DecodeFrameParam; using ResetParam = H264PingInfoParser::ResetParam; using GetImageParam = H264PingInfoParser::GetImageParam; MediaH264DecoderFfmpeg::~MediaH264DecoderFfmpeg() { H264_DPRINT("destroyed MediaH264DecoderFfmpeg %p", this); destroyH264Context(); } void MediaH264DecoderFfmpeg::reset(void* ptr) { destroyH264Context(); ResetParam param{}; mParser.parseResetParams(ptr, param); initH264ContextInternal(param.width, param.height, param.outputWidth, param.outputHeight, param.outputPixelFormat); } void MediaH264DecoderFfmpeg::initH264Context(void* ptr) { InitContextParam param{}; mParser.parseInitContextParams(ptr, param); initH264ContextInternal(param.width, param.height, param.outputWidth, param.outputHeight, param.outputPixelFormat); } void MediaH264DecoderFfmpeg::initH264ContextInternal(unsigned int width, unsigned int height, unsigned int outWidth, unsigned int outHeight, PixelFormat outPixFmt) { H264_DPRINT("%s(w=%u h=%u out_w=%u out_h=%u pixfmt=%u)", __func__, width, height, outWidth, outHeight, (uint8_t)outPixFmt); mWidth = width; mHeight = height; mOutputWidth = outWidth; mOutputHeight = outHeight; mOutPixFmt = outPixFmt; mOutBufferSize = outWidth * outHeight * 3 / 2; mIsInFlush = false; mDecodedFrame.resize(mOutBufferSize); // standard ffmpeg codec stuff avcodec_register_all(); if(0){ AVCodec* current_codec = NULL; current_codec = av_codec_next(current_codec); while (current_codec != NULL) { if (av_codec_is_decoder(current_codec)) { H264_DPRINT("codec decoder found %s long name %s", current_codec->name, current_codec->long_name); } current_codec = av_codec_next(current_codec); } } mCodec = NULL; auto useCuvidEnv = android::base::System::getEnvironmentVariable( "ANDROID_EMU_CODEC_USE_FFMPEG_CUVID_DECODER"); if (useCuvidEnv != "") { mCodec = avcodec_find_decoder_by_name("h264_cuvid"); if (mCodec) { mIsSoftwareDecoder = false; H264_DPRINT("Found h264_cuvid decoder, using it"); } else { H264_DPRINT("Cannot find h264_cuvid decoder"); } } if (!mCodec) { mCodec = avcodec_find_decoder(AV_CODEC_ID_H264); H264_DPRINT("Using default software h264 decoder"); } mCodecCtx = avcodec_alloc_context3(mCodec); mCodecCtx->thread_count = 4; mCodecCtx->thread_type = FF_THREAD_FRAME; avcodec_open2(mCodecCtx, mCodec, 0); mFrame = av_frame_alloc(); H264_DPRINT("Successfully created software h264 decoder context %p", mCodecCtx); } MediaH264DecoderPlugin* MediaH264DecoderFfmpeg::clone() { H264_DPRINT("clone MediaH264DecoderFfmpeg %p with version %d", this, (int)mParser.version()); return new MediaH264DecoderFfmpeg(mId, mParser); } MediaH264DecoderFfmpeg::MediaH264DecoderFfmpeg(uint64_t id, H264PingInfoParser parser) : mId(id), mParser(parser) { H264_DPRINT("allocated MediaH264DecoderFfmpeg %p with version %d", this, (int)mParser.version()); } void MediaH264DecoderFfmpeg::destroyH264Context() { H264_DPRINT("Destroy context %p", this); if (mCodecCtx) { avcodec_close(mCodecCtx); av_free(mCodecCtx); mCodecCtx = NULL; } if (mFrame) { av_frame_free(&mFrame); mFrame = NULL; } } void MediaH264DecoderFfmpeg::resetDecoder() { mNumDecodedFrame = 0; avcodec_close(mCodecCtx); av_free(mCodecCtx); mCodecCtx = avcodec_alloc_context3(mCodec); avcodec_open2(mCodecCtx, mCodec, 0); } bool MediaH264DecoderFfmpeg::checkWhetherConfigChanged(const uint8_t* frame, size_t szBytes) { // get frame type // if the frame is none SPS/PPS, return false // otherwise, check both SPS/PPS and return true const uint8_t* currNalu = H264NaluParser::getNextStartCodeHeader(frame, szBytes); if (currNalu == nullptr) { // should not happen H264_DPRINT("Found bad frame"); return false; } size_t remaining = szBytes - (currNalu - frame); size_t currNaluSize = remaining; H264NaluParser::H264NaluType currNaluType = H264NaluParser::getFrameNaluType(currNalu, currNaluSize, NULL); if (currNaluType != H264NaluParser::H264NaluType::SPS) { return false; } H264_DPRINT("found SPS\n"); const uint8_t* nextNalu = H264NaluParser::getNextStartCodeHeader(currNalu + 3, remaining - 3); if (nextNalu == nullptr) { // only one nalu, cannot have configuration change H264_DPRINT("frame has only one Nalu unit, cannot be configuration change\n"); return false; } if (mNumDecodedFrame == 0) { H264_DPRINT("have not decoded anything yet, cannot be config change"); return false; } // pretty sure it is config change H264_DPRINT("\n\nDetected stream configuration change !!!\n\n"); return true; } void MediaH264DecoderFfmpeg::decodeFrameDirect(void* ptr, const uint8_t* frame, size_t szBytes, uint64_t pts) { DecodeFrameParam param{}; mParser.parseDecodeFrameParams(ptr, param); param.pData = (uint8_t*)frame; param.pts = pts; param.size = szBytes; decodeFrameInternal(param); } void MediaH264DecoderFfmpeg::decodeFrame(void* ptr) { DecodeFrameParam param{}; mParser.parseDecodeFrameParams(ptr, param); decodeFrameInternal(param); } void MediaH264DecoderFfmpeg::decodeFrameInternal(DecodeFrameParam& param) { const uint8_t* frame = param.pData; size_t szBytes = param.size; uint64_t inputPts = param.pts; H264_DPRINT("%s(frame=%p, sz=%zu pts %lld)", __func__, frame, szBytes, (long long)inputPts); Err h264Err = Err::NoErr; // TODO: move this somewhere else // First return parameter is the number of bytes processed, // Second return parameter is the error code size_t* retSzBytes = param.pConsumedBytes; int32_t* retErr = param.pDecoderErrorCode; const bool enableSnapshot = true; if (enableSnapshot) { std::vector v; v.assign(frame, frame + szBytes); bool hasSps = H264NaluParser::checkSpsFrame(frame, szBytes); if (hasSps) { mSnapshotState = SnapshotState{}; mSnapshotState.saveSps(v); } else { bool hasPps = H264NaluParser::checkPpsFrame(frame, szBytes); if (hasPps) { mSnapshotState.savePps(v); mSnapshotState.savedPackets.clear(); mSnapshotState.savedDecodedFrame.data.clear(); } else { bool isIFrame = H264NaluParser::checkIFrame(frame, szBytes); if (isIFrame) { mSnapshotState.savedPackets.clear(); } const bool saveOK = mSnapshotState.savePacket(std::move(v), inputPts); if (saveOK) { H264_DPRINT("saving packet; total is %d", (int)(mSnapshotState.savedPackets.size())); } else { H264_DPRINT("saving packet; has duplicate, skip; total is %d", (int)(mSnapshotState.savedPackets.size())); *retSzBytes = szBytes; *retErr = (int32_t)h264Err; mImageReady = true; return; } } } } if (!mIsSoftwareDecoder) { bool configChanged = checkWhetherConfigChanged(frame, szBytes); if (configChanged) { resetDecoder(); } } av_init_packet(&mPacket); mPacket.data = (unsigned char*)frame; mPacket.size = szBytes; mPacket.pts = inputPts; avcodec_send_packet(mCodecCtx, &mPacket); int retframe = avcodec_receive_frame(mCodecCtx, mFrame); *retSzBytes = szBytes; *retErr = (int32_t)h264Err; mIsInFlush = false; if (retframe != 0) { H264_DPRINT("decodeFrame has nonzero return value %d", retframe); if (retframe == AVERROR_EOF) { H264_DPRINT("EOF returned from decoder"); H264_DPRINT("EOF returned from decoder reset context now"); resetDecoder(); } else if (retframe == AVERROR(EAGAIN)) { H264_DPRINT("EAGAIN returned from decoder"); } else { H264_DPRINT("unknown value %d", retframe); } return; } H264_DPRINT("new w %d new h %d, old w %d old h %d", mFrame->width, mFrame->height, mOutputWidth, mOutputHeight); mFrameFormatChanged = false; ++mNumDecodedFrame; copyFrame(); H264_DPRINT("%s: got frame in decode mode", __func__); mImageReady = true; } void MediaH264DecoderFfmpeg::copyFrame() { int w = mFrame->width; int h = mFrame->height; if (w != mOutputWidth || h != mOutputHeight) { mOutputWidth = w; mOutputHeight= h; mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2; mDecodedFrame.resize(mOutBufferSize); } H264_DPRINT("w %d h %d Y line size %d U line size %d V line size %d", w, h, mFrame->linesize[0], mFrame->linesize[1], mFrame->linesize[2]); for (int i = 0; i < h; ++i) { memcpy(mDecodedFrame.data() + i * w, mFrame->data[0] + i * mFrame->linesize[0], w); } H264_DPRINT("format is %d and NV21 is %d 12 is %d", mFrame->format, (int)AV_PIX_FMT_NV21, (int)AV_PIX_FMT_NV12); if (mFrame->format == AV_PIX_FMT_NV12) { for (int i=0; i < h / 2; ++i) { memcpy(w * h + mDecodedFrame.data() + i * w, mFrame->data[1] + i * mFrame->linesize[1], w); } YuvConverter convert8(mOutputWidth, mOutputHeight); convert8.UVInterleavedToPlanar(mDecodedFrame.data()); } else { for (int i=0; i < h / 2; ++i) { memcpy(w * h + mDecodedFrame.data() + i * w / 2, mFrame->data[1] + i * mFrame->linesize[1], w / 2); } for (int i=0; i < h / 2; ++i) { memcpy(w * h + w * h / 4 + mDecodedFrame.data() + i * w / 2, mFrame->data[2] + i * mFrame->linesize[2], w / 2); } } mColorPrimaries = mFrame->color_primaries; mColorRange = mFrame->color_range; mColorTransfer = mFrame->color_trc; mColorSpace = mFrame->colorspace; mOutputPts = mFrame->pts; H264_DPRINT("copied Frame and it has presentation time at %lld", (long long)(mFrame->pts)); H264_DPRINT("Frame primary %d range %d transfer %d space %d", mFrame->color_primaries, mFrame->color_range, mFrame->color_trc, mFrame->colorspace); } void MediaH264DecoderFfmpeg::flush(void* ptr) { H264_DPRINT("Flushing..."); mIsInFlush = true; mSnapshotState = SnapshotState{}; H264_DPRINT("Flushing done"); } void MediaH264DecoderFfmpeg::getImage(void* ptr) { H264_DPRINT("getImage %p", ptr); GetImageParam param{}; mParser.parseGetImageParams(ptr, param); int* retErr = param.pDecoderErrorCode; uint32_t* retWidth = param.pRetWidth; uint32_t* retHeight = param.pRetHeight; uint64_t* retPts = param.pRetPts; uint32_t* retColorPrimaries = param.pRetColorPrimaries; uint32_t* retColorRange = param.pRetColorRange; uint32_t* retColorTransfer = param.pRetColorTransfer; uint32_t* retColorSpace = param.pRetColorSpace; static int numbers=0; //H264_DPRINT("calling getImage %d", numbers++); if (!mImageReady) { if (mFrameFormatChanged) { *retWidth = mOutputWidth; *retHeight = mOutputHeight; *retErr = static_cast(Err::DecoderRestarted); return; } if (mIsInFlush) { // guest be in flush mode, so try to get a frame avcodec_send_packet(mCodecCtx, NULL); int retframe = avcodec_receive_frame(mCodecCtx, mFrame); if (retframe == AVERROR(EAGAIN) || retframe == AVERROR_EOF) { H264_DPRINT("%s: frame is null", __func__); *retErr = static_cast(Err::NoDecodedFrame); return; } if (retframe != 0) { char tmp[1024]; av_strerror(retframe, tmp, sizeof(tmp)); H264_DPRINT("WARNING: some unknown error %d: %s", retframe, tmp); *retErr = static_cast(Err::NoDecodedFrame); return; } H264_DPRINT("%s: got frame in flush mode retrun code %d", __func__, retframe); //now copy to mDecodedFrame copyFrame(); mImageReady = true; } else { H264_DPRINT("%s: no new frame yet", __func__); *retErr = static_cast(Err::NoDecodedFrame); return; } } *retWidth = mOutputWidth; *retHeight = mOutputHeight; *retPts = mOutputPts; *retColorPrimaries = mColorPrimaries; *retColorRange = mColorRange; *retColorTransfer = mColorTransfer; *retColorSpace = mColorSpace; if (mParser.version() == 100) { uint8_t* dst = param.pDecodedFrame; memcpy(dst, mDecodedFrame.data(), mOutBufferSize); } else if (mParser.version() == 200) { if (param.hostColorBufferId >= 0) { mRenderer.renderToHostColorBuffer(param.hostColorBufferId, mOutputWidth, mOutputHeight, mDecodedFrame.data()); } else { uint8_t* dst = param.pDecodedFrame; memcpy(dst, mDecodedFrame.data(), mOutBufferSize); } } mImageReady = false; *retErr = mOutBufferSize; H264_DPRINT("getImage %p done", ptr); } void MediaH264DecoderFfmpeg::save(base::Stream* stream) const { stream->putBe32(mParser.version()); stream->putBe32(mWidth); stream->putBe32(mHeight); stream->putBe32((int)mOutPixFmt); const int hasContext = mCodecCtx != nullptr ? 1 : 0; stream->putBe32(hasContext); if (mImageReady) { mSnapshotState.saveDecodedFrame( mDecodedFrame, mOutputWidth, mOutputHeight, ColorAspects{mColorPrimaries, mColorRange, mColorTransfer, mColorSpace}, mOutputPts); } else { mSnapshotState.savedDecodedFrame.data.clear(); } H264_DPRINT("saving packets now %d", (int)(mSnapshotState.savedPackets.size())); mSnapshotState.save(stream); } void MediaH264DecoderFfmpeg::oneShotDecode(std::vector& data, uint64_t pts) { av_init_packet(&mPacket); mPacket.data = (unsigned char*)(data.data()); mPacket.size = data.size(); mPacket.pts = pts; H264_DPRINT("decoding pts %lld packet size %d", (long long)pts, (int)data.size()); avcodec_send_packet(mCodecCtx, &mPacket); avcodec_receive_frame(mCodecCtx, mFrame); } bool MediaH264DecoderFfmpeg::load(base::Stream* stream) { uint32_t version = stream->getBe32(); mParser = H264PingInfoParser{version}; mWidth = stream->getBe32(); mHeight = stream->getBe32(); mOutPixFmt = (PixelFormat)stream->getBe32(); const int hasContext = stream->getBe32(); if (hasContext) { initH264ContextInternal(mWidth, mHeight, mWidth, mHeight, mOutPixFmt); assert(mCodecCtx != nullptr); } mSnapshotState.load(stream); H264_DPRINT("loaded packets %d, now restore decoder", (int)(mSnapshotState.savedPackets.size())); if (hasContext && mCodecCtx != nullptr && mSnapshotState.sps.size() > 0) { oneShotDecode(mSnapshotState.sps, 0); if (mSnapshotState.pps.size() > 0) { oneShotDecode(mSnapshotState.pps, 0); if (mSnapshotState.savedPackets.size() > 0) { for (int i = 0; i < mSnapshotState.savedPackets.size(); ++i) { PacketInfo& pkt = mSnapshotState.savedPackets[i]; oneShotDecode(pkt.data, pkt.pts); } copyFrame(); //save the last frame } } } if (mSnapshotState.savedDecodedFrame.data.size() > 0) { mDecodedFrame = mSnapshotState.savedDecodedFrame.data; mOutBufferSize = mSnapshotState.savedDecodedFrame.data.size(); mOutputWidth = mSnapshotState.savedDecodedFrame.width; mOutputHeight = mSnapshotState.savedDecodedFrame.height; mColorPrimaries = mSnapshotState.savedDecodedFrame.color.primaries; mColorRange = mSnapshotState.savedDecodedFrame.color.range; mColorTransfer = mSnapshotState.savedDecodedFrame.color.transfer; mColorSpace = mSnapshotState.savedDecodedFrame.color.space; mOutputPts = mSnapshotState.savedDecodedFrame.pts; mImageReady = true; } else { mImageReady = false; } H264_DPRINT("Done loading snapshots frames\n\n"); return true; } } // namespace emulation } // namespace android