1 // Copyright (C) 2020 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "host-common/MediaH264DecoderGeneric.h"
16 #include "base/System.h"
17 #include "host-common/H264PingInfoParser.h"
18 #include "host-common/MediaFfmpegVideoHelper.h"
19 #include "android/main-emugl.h"
20
21 #ifndef __APPLE__
22 // for Linux and Window, Cuvid is available
23 #include "host-common/MediaCudaDriverHelper.h"
24 #include "host-common/MediaCudaVideoHelper.h"
25 #endif
26
27 #include <cstdint>
28 #include <string>
29 #include <vector>
30
31 #include <stdio.h>
32 #include <string.h>
33
34 #define MEDIA_H264_DEBUG 0
35
36 #if MEDIA_H264_DEBUG
37 #define H264_DPRINT(fmt, ...) \
38 fprintf(stderr, "h264-dec-generic: %s:%d " fmt "\n", __func__, __LINE__, \
39 ##__VA_ARGS__);
40 #else
41 #define H264_DPRINT(fmt, ...)
42 #endif
43
44 namespace android {
45 namespace emulation {
46
47 using InitContextParam = H264PingInfoParser::InitContextParam;
48 using DecodeFrameParam = H264PingInfoParser::DecodeFrameParam;
49 using ResetParam = H264PingInfoParser::ResetParam;
50 using GetImageParam = H264PingInfoParser::GetImageParam;
51 using TextureFrame = MediaHostRenderer::TextureFrame;
52
53 namespace {
54
canUseCudaDecoder()55 bool canUseCudaDecoder() {
56 #ifndef __APPLE__
57 if (MediaCudaDriverHelper::initCudaDrivers()) {
58 H264_DPRINT("Using Cuvid decoder on Linux/Windows");
59 return true;
60 } else {
61 H264_DPRINT(
62 "ERROR: cannot use cuvid decoder: failed to init cuda driver");
63 return false;
64 }
65 #else
66 return false;
67 #endif
68 }
69
canDecodeToGpuTexture()70 bool canDecodeToGpuTexture() {
71 #ifndef __APPLE__
72 if (emuglConfig_get_current_renderer() == SELECTED_RENDERER_HOST) {
73 return true;
74 } else {
75 return false;
76 }
77 #else
78 return false;
79 #endif
80 }
81 }; // end namespace
82
MediaH264DecoderGeneric(uint64_t id,H264PingInfoParser parser)83 MediaH264DecoderGeneric::MediaH264DecoderGeneric(uint64_t id,
84 H264PingInfoParser parser)
85 : mId(id), mParser(parser) {
86 H264_DPRINT("allocated MediaH264DecoderGeneric %p with version %d", this,
87 (int)mParser.version());
88 mUseGpuTexture = canDecodeToGpuTexture();
89 }
90
~MediaH264DecoderGeneric()91 MediaH264DecoderGeneric::~MediaH264DecoderGeneric() {
92 H264_DPRINT("destroyed MediaH264DecoderGeneric %p", this);
93 destroyH264Context();
94 }
95
reset(void * ptr)96 void MediaH264DecoderGeneric::reset(void* ptr) {
97 destroyH264Context();
98 ResetParam param{};
99 mParser.parseResetParams(ptr, param);
100 initH264ContextInternal(param.width, param.height, param.outputWidth,
101 param.outputHeight, param.outputPixelFormat);
102 }
103
initH264Context(void * ptr)104 void MediaH264DecoderGeneric::initH264Context(void* ptr) {
105 InitContextParam param{};
106 mParser.parseInitContextParams(ptr, param);
107 initH264ContextInternal(param.width, param.height, param.outputWidth,
108 param.outputHeight, param.outputPixelFormat);
109 }
110
initH264ContextInternal(unsigned int width,unsigned int height,unsigned int outWidth,unsigned int outHeight,PixelFormat outPixFmt)111 void MediaH264DecoderGeneric::initH264ContextInternal(unsigned int width,
112 unsigned int height,
113 unsigned int outWidth,
114 unsigned int outHeight,
115 PixelFormat outPixFmt) {
116 H264_DPRINT("%s(w=%u h=%u out_w=%u out_h=%u pixfmt=%u)", __func__, width,
117 height, outWidth, outHeight, (uint8_t)outPixFmt);
118 mWidth = width;
119 mHeight = height;
120 mOutputWidth = outWidth;
121 mOutputHeight = outHeight;
122 mOutPixFmt = outPixFmt;
123
124 #ifndef __APPLE__
125 if (canUseCudaDecoder() && mParser.version() >= 200) {
126 MediaCudaVideoHelper::OutputTreatmentMode oMode =
127 MediaCudaVideoHelper::OutputTreatmentMode::SAVE_RESULT;
128
129 MediaCudaVideoHelper::FrameStorageMode fMode =
130 mUseGpuTexture ? MediaCudaVideoHelper::FrameStorageMode::
131 USE_GPU_TEXTURE
132 : MediaCudaVideoHelper::FrameStorageMode::
133 USE_BYTE_BUFFER;
134
135 auto cudavid =
136 new MediaCudaVideoHelper(oMode, fMode, cudaVideoCodec_H264);
137
138 if (mUseGpuTexture) {
139 H264_DPRINT("use gpu texture");
140 cudavid->resetTexturePool(mRenderer.getTexturePool());
141 }
142 mHwVideoHelper.reset(cudavid);
143 if (!mHwVideoHelper->init()) {
144 mHwVideoHelper.reset(nullptr);
145 H264_DPRINT("failed to init cuda decoder");
146 } else {
147 H264_DPRINT("succeeded to init cuda decoder");
148 }
149 }
150 // TODO: add video toolbox for apple
151 #endif
152
153 mSnapshotHelper.reset(
154 new MediaSnapshotHelper(MediaSnapshotHelper::CodecType::H264));
155
156 H264_DPRINT("Successfully created h264 decoder context %p", this);
157 }
158
createAndInitSoftVideoHelper()159 void MediaH264DecoderGeneric::createAndInitSoftVideoHelper() {
160 mSwVideoHelper.reset(
161 new MediaFfmpegVideoHelper(264, mParser.version() < 200 ? 1 : 4));
162 mUseGpuTexture = false;
163 mSwVideoHelper->init();
164 }
165
clone()166 MediaH264DecoderPlugin* MediaH264DecoderGeneric::clone() {
167 H264_DPRINT("clone MediaH264DecoderGeneric %p with version %d", this,
168 (int)mParser.version());
169 return new MediaH264DecoderGeneric(mId, mParser);
170 }
171
destroyH264Context()172 void MediaH264DecoderGeneric::destroyH264Context() {
173 H264_DPRINT("Destroy context %p", this);
174 while (mSnapshotHelper != nullptr) {
175 MediaSnapshotState::FrameInfo* pFrame = mSnapshotHelper->frontFrame();
176 if (!pFrame) {
177 break;
178 }
179 if (mUseGpuTexture && pFrame->texture[0] > 0 &&
180 pFrame->texture[1] > 0) {
181 mRenderer.putTextureFrame(
182 TextureFrame{pFrame->texture[0], pFrame->texture[1]});
183 }
184 mSnapshotHelper->discardFrontFrame();
185 }
186 mRenderer.cleanUpTextures();
187
188 if (mHwVideoHelper != nullptr) {
189 mHwVideoHelper->deInit();
190 mHwVideoHelper.reset(nullptr);
191 }
192 if (mVideoHelper != nullptr) {
193 mVideoHelper->deInit();
194 mVideoHelper.reset(nullptr);
195 }
196 }
197
decodeFrame(void * ptr)198 void MediaH264DecoderGeneric::decodeFrame(void* ptr) {
199 DecodeFrameParam param{};
200 mParser.parseDecodeFrameParams(ptr, param);
201 const uint8_t* frame = param.pData;
202 size_t szBytes = param.size;
203 uint64_t inputPts = param.pts;
204
205 H264_DPRINT("%s(frame=%p, sz=%zu pts %lld)", __func__, frame, szBytes,
206 (long long)inputPts);
207 size_t* retSzBytes = param.pConsumedBytes;
208 int32_t* retErr = param.pDecoderErrorCode;
209
210 decodeFrameInternal(frame, szBytes, inputPts);
211
212 mSnapshotHelper->savePacket(frame, szBytes, inputPts);
213 fetchAllFrames();
214
215 *retSzBytes = szBytes;
216 *retErr = (int32_t)Err::NoErr;
217 }
218
decodeFrameInternal(const uint8_t * data,size_t len,uint64_t pts)219 void MediaH264DecoderGeneric::decodeFrameInternal(const uint8_t* data,
220 size_t len,
221 uint64_t pts) {
222 if (mTrialPeriod) {
223 try_decode(data, len, pts);
224 } else {
225 mVideoHelper->decode(data, len, pts);
226 }
227 }
228
try_decode(const uint8_t * data,size_t len,uint64_t pts)229 void MediaH264DecoderGeneric::try_decode(const uint8_t* data,
230 size_t len,
231 uint64_t pts) {
232 // for h264, it needs sps/pps to decide whether decoding is
233 // possible;
234 if (mHwVideoHelper != nullptr) {
235 mHwVideoHelper->decode(data, len, pts);
236 if (mHwVideoHelper->good()) {
237 return;
238 } else {
239 H264_DPRINT("Failed to decode with HW decoder %d; switch to SW",
240 mHwVideoHelper->error());
241 mHwVideoHelper.reset(nullptr);
242 }
243 }
244
245 createAndInitSoftVideoHelper();
246 mVideoHelper = std::move(mSwVideoHelper);
247
248 mVideoHelper->setIgnoreDecodedFrames();
249 std::function<void(const uint8_t*, size_t, uint64_t)> func =
250 [=](const uint8_t* data, size_t len, uint64_t pts) {
251 this->oneShotDecode(data, len, pts);
252 };
253
254 mSnapshotHelper->replay(func);
255
256 mVideoHelper->setSaveDecodedFrames();
257
258 mVideoHelper->decode(data, len, pts);
259 mTrialPeriod = false;
260 }
261
fetchAllFrames()262 void MediaH264DecoderGeneric::fetchAllFrames() {
263 while (true) {
264 MediaSnapshotState::FrameInfo frame;
265 bool success =
266 mHwVideoHelper
267 ? mHwVideoHelper->receiveFrame(&frame)
268 : (mVideoHelper ? mVideoHelper->receiveFrame(&frame)
269 : false);
270 if (!success) {
271 break;
272 }
273 mSnapshotHelper->saveDecodedFrame(std::move(frame));
274 }
275 }
276
flush(void * ptr)277 void MediaH264DecoderGeneric::flush(void* ptr) {
278 H264_DPRINT("Flushing...");
279 if (mHwVideoHelper) {
280 mHwVideoHelper->flush();
281 } else if (mVideoHelper) {
282 mVideoHelper->flush();
283 }
284 fetchAllFrames();
285 H264_DPRINT("Flushing done");
286 }
287
getImage(void * ptr)288 void MediaH264DecoderGeneric::getImage(void* ptr) {
289 H264_DPRINT("getImage %p", ptr);
290 GetImageParam param{};
291 mParser.parseGetImageParams(ptr, param);
292
293 int* retErr = param.pDecoderErrorCode;
294 uint32_t* retWidth = param.pRetWidth;
295 uint32_t* retHeight = param.pRetHeight;
296 uint64_t* retPts = param.pRetPts;
297 uint32_t* retColorPrimaries = param.pRetColorPrimaries;
298 uint32_t* retColorRange = param.pRetColorRange;
299 uint32_t* retColorTransfer = param.pRetColorTransfer;
300 uint32_t* retColorSpace = param.pRetColorSpace;
301 uint8_t* dst = param.pDecodedFrame;
302
303 static int numbers = 0;
304 H264_DPRINT("calling getImage %d colorbuffer %d", numbers++,
305 (int)param.hostColorBufferId);
306
307 MediaSnapshotState::FrameInfo* pFrame = mSnapshotHelper->frontFrame();
308 if (pFrame == nullptr) {
309 H264_DPRINT("there is no image");
310 *retErr = static_cast<int>(Err::NoDecodedFrame);
311 return;
312 }
313
314 bool needToCopyToGuest = true;
315 if (mParser.version() == 200) {
316 if (mUseGpuTexture && pFrame->texture[0] > 0 && pFrame->texture[1] > 0) {
317 mRenderer.renderToHostColorBufferWithTextures(
318 param.hostColorBufferId, pFrame->width, pFrame->height,
319 TextureFrame{pFrame->texture[0], pFrame->texture[1]});
320 } else {
321 H264_DPRINT(
322 "calling rendering to host side color buffer with id %d",
323 param.hostColorBufferId);
324 mRenderer.renderToHostColorBuffer(param.hostColorBufferId,
325 pFrame->width, pFrame->height,
326 pFrame->data.data());
327 }
328 needToCopyToGuest = false;
329 }
330
331 if (needToCopyToGuest) {
332 memcpy(param.pDecodedFrame, pFrame->data.data(),
333 pFrame->width * pFrame->height * 3 / 2);
334 }
335
336 *retErr = pFrame->width * pFrame->height * 3 / 2;
337 *retWidth = pFrame->width;
338 *retHeight = pFrame->height;
339 *retPts = pFrame->pts;
340 *retColorPrimaries = pFrame->color.primaries;
341 *retColorRange = pFrame->color.range;
342 *retColorSpace = pFrame->color.space;
343 *retColorTransfer = pFrame->color.transfer;
344
345 mSnapshotHelper->discardFrontFrame();
346
347 H264_DPRINT("Copying completed pts %lld w %d h %d ow %d oh %d",
348 (long long)*retPts, (int)*retWidth, (int)*retHeight,
349 (int)mOutputWidth, (int)mOutputHeight);
350 }
351
save(base::Stream * stream) const352 void MediaH264DecoderGeneric::save(base::Stream* stream) const {
353 stream->putBe32(mParser.version());
354 stream->putBe32(mWidth);
355 stream->putBe32(mHeight);
356 stream->putBe32((int)mOutPixFmt);
357
358 const int hasContext = (mVideoHelper || mHwVideoHelper) ? 1 : 0;
359 stream->putBe32(hasContext);
360
361 mSnapshotHelper->save(stream);
362 }
363
oneShotDecode(const uint8_t * data,size_t len,uint64_t pts)364 void MediaH264DecoderGeneric::oneShotDecode(const uint8_t* data,
365 size_t len,
366 uint64_t pts) {
367 if (!mHwVideoHelper && !mSwVideoHelper && !mVideoHelper) {
368 return;
369 }
370
371 decodeFrameInternal(data, len, pts);
372 while (true) {
373 MediaSnapshotState::FrameInfo frame;
374 bool success = mHwVideoHelper ? mHwVideoHelper->receiveFrame(&frame)
375 : mVideoHelper->receiveFrame(&frame);
376 if (!success) {
377 break;
378 }
379 }
380 }
381
load(base::Stream * stream)382 bool MediaH264DecoderGeneric::load(base::Stream* stream) {
383 uint32_t version = stream->getBe32();
384 mParser = H264PingInfoParser{version};
385
386 mWidth = stream->getBe32();
387 mHeight = stream->getBe32();
388 mOutPixFmt = (PixelFormat)stream->getBe32();
389
390 const int hasContext = stream->getBe32();
391 if (hasContext) {
392 initH264ContextInternal(mWidth, mHeight, mWidth, mHeight, mOutPixFmt);
393 }
394
395 if (mVideoHelper) {
396 mVideoHelper->setIgnoreDecodedFrames();
397 }
398 std::function<void(const uint8_t*, size_t, uint64_t)> func =
399 [=](const uint8_t* data, size_t len, uint64_t pts) {
400 this->oneShotDecode(data, len, pts);
401 };
402
403 mSnapshotHelper->load(stream, func);
404
405 if (mVideoHelper) {
406 mVideoHelper->setSaveDecodedFrames();
407 }
408
409 H264_DPRINT("Done loading snapshots frames\n\n");
410 return true;
411 }
412
413 } // namespace emulation
414 } // namespace android
415