1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Camera3-HeicCompositeStream"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
20 //#define LOG_NDEBUG 0
21
22 #include <linux/memfd.h>
23 #include <pthread.h>
24 #include <sys/syscall.h>
25
26 #include <aidl/android/hardware/camera/device/CameraBlob.h>
27 #include <aidl/android/hardware/camera/device/CameraBlobId.h>
28 #include <libyuv.h>
29 #include <gui/Surface.h>
30 #include <utils/Log.h>
31 #include <utils/Trace.h>
32
33 #include <mediadrm/ICrypto.h>
34 #include <media/MediaCodecBuffer.h>
35 #include <media/stagefright/foundation/ABuffer.h>
36 #include <media/stagefright/foundation/MediaDefs.h>
37 #include <media/stagefright/MediaCodecConstants.h>
38
39 #include "common/CameraDeviceBase.h"
40 #include "utils/ExifUtils.h"
41 #include "utils/SessionConfigurationUtils.h"
42 #include "HeicEncoderInfoManager.h"
43 #include "HeicCompositeStream.h"
44
45 using aidl::android::hardware::camera::device::CameraBlob;
46 using aidl::android::hardware::camera::device::CameraBlobId;
47
48 namespace android {
49 namespace camera3 {
50
HeicCompositeStream(sp<CameraDeviceBase> device,wp<hardware::camera2::ICameraDeviceCallbacks> cb)51 HeicCompositeStream::HeicCompositeStream(sp<CameraDeviceBase> device,
52 wp<hardware::camera2::ICameraDeviceCallbacks> cb) :
53 CompositeStream(device, cb),
54 mUseHeic(false),
55 mNumOutputTiles(1),
56 mOutputWidth(0),
57 mOutputHeight(0),
58 mMaxHeicBufferSize(0),
59 mGridWidth(HeicEncoderInfoManager::kGridWidth),
60 mGridHeight(HeicEncoderInfoManager::kGridHeight),
61 mGridRows(1),
62 mGridCols(1),
63 mUseGrid(false),
64 mAppSegmentStreamId(-1),
65 mAppSegmentSurfaceId(-1),
66 mMainImageStreamId(-1),
67 mMainImageSurfaceId(-1),
68 mYuvBufferAcquired(false),
69 mProducerListener(new ProducerListener()),
70 mDequeuedOutputBufferCnt(0),
71 mCodecOutputCounter(0),
72 mQuality(-1),
73 mGridTimestampUs(0),
74 mStatusId(StatusTracker::NO_STATUS_ID) {
75 }
76
~HeicCompositeStream()77 HeicCompositeStream::~HeicCompositeStream() {
78 // Call deinitCodec in case stream hasn't been deleted yet to avoid any
79 // memory/resource leak.
80 deinitCodec();
81
82 mInputAppSegmentBuffers.clear();
83 mCodecOutputBuffers.clear();
84
85 mAppSegmentStreamId = -1;
86 mAppSegmentSurfaceId = -1;
87 mAppSegmentConsumer.clear();
88 mAppSegmentSurface.clear();
89
90 mMainImageStreamId = -1;
91 mMainImageSurfaceId = -1;
92 mMainImageConsumer.clear();
93 mMainImageSurface.clear();
94 }
95
isHeicCompositeStream(const sp<Surface> & surface)96 bool HeicCompositeStream::isHeicCompositeStream(const sp<Surface> &surface) {
97 ANativeWindow *anw = surface.get();
98 status_t err;
99 int format;
100 if ((err = anw->query(anw, NATIVE_WINDOW_FORMAT, &format)) != OK) {
101 String8 msg = String8::format("Failed to query Surface format: %s (%d)", strerror(-err),
102 err);
103 ALOGE("%s: %s", __FUNCTION__, msg.string());
104 return false;
105 }
106
107 int dataspace;
108 if ((err = anw->query(anw, NATIVE_WINDOW_DEFAULT_DATASPACE, &dataspace)) != OK) {
109 String8 msg = String8::format("Failed to query Surface dataspace: %s (%d)", strerror(-err),
110 err);
111 ALOGE("%s: %s", __FUNCTION__, msg.string());
112 return false;
113 }
114
115 return ((format == HAL_PIXEL_FORMAT_BLOB) && (dataspace == HAL_DATASPACE_HEIF));
116 }
117
createInternalStreams(const std::vector<sp<Surface>> & consumers,bool,uint32_t width,uint32_t height,int format,camera_stream_rotation_t rotation,int * id,const String8 & physicalCameraId,const std::unordered_set<int32_t> & sensorPixelModesUsed,std::vector<int> * surfaceIds,int,bool,int32_t colorSpace,int64_t,int64_t,bool useReadoutTimestamp)118 status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface>>& consumers,
119 bool /*hasDeferredConsumer*/, uint32_t width, uint32_t height, int format,
120 camera_stream_rotation_t rotation, int *id, const String8& physicalCameraId,
121 const std::unordered_set<int32_t> &sensorPixelModesUsed,
122 std::vector<int> *surfaceIds,
123 int /*streamSetId*/, bool /*isShared*/, int32_t colorSpace,
124 int64_t /*dynamicProfile*/, int64_t /*streamUseCase*/, bool useReadoutTimestamp) {
125 sp<CameraDeviceBase> device = mDevice.promote();
126 if (!device.get()) {
127 ALOGE("%s: Invalid camera device!", __FUNCTION__);
128 return NO_INIT;
129 }
130
131 status_t res = initializeCodec(width, height, device);
132 if (res != OK) {
133 ALOGE("%s: Failed to initialize HEIC/HEVC codec: %s (%d)",
134 __FUNCTION__, strerror(-res), res);
135 return NO_INIT;
136 }
137
138 sp<IGraphicBufferProducer> producer;
139 sp<IGraphicBufferConsumer> consumer;
140 BufferQueue::createBufferQueue(&producer, &consumer);
141 mAppSegmentConsumer = new CpuConsumer(consumer, kMaxAcquiredAppSegment);
142 mAppSegmentConsumer->setFrameAvailableListener(this);
143 mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));
144 mAppSegmentSurface = new Surface(producer);
145
146 mStaticInfo = device->info();
147
148 res = device->createStream(mAppSegmentSurface, mAppSegmentMaxSize, 1, format,
149 kAppSegmentDataSpace, rotation, &mAppSegmentStreamId, physicalCameraId,
150 sensorPixelModesUsed, surfaceIds, camera3::CAMERA3_STREAM_SET_ID_INVALID,
151 /*isShared*/false, /*isMultiResolution*/false,
152 /*consumerUsage*/0, ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
153 ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
154 OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
155 OutputConfiguration::MIRROR_MODE_AUTO,
156 colorSpace,
157 useReadoutTimestamp);
158 if (res == OK) {
159 mAppSegmentSurfaceId = (*surfaceIds)[0];
160 } else {
161 ALOGE("%s: Failed to create JPEG App segment stream: %s (%d)", __FUNCTION__,
162 strerror(-res), res);
163 return res;
164 }
165
166 if (!mUseGrid) {
167 res = mCodec->createInputSurface(&producer);
168 if (res != OK) {
169 ALOGE("%s: Failed to create input surface for Heic codec: %s (%d)",
170 __FUNCTION__, strerror(-res), res);
171 return res;
172 }
173 } else {
174 BufferQueue::createBufferQueue(&producer, &consumer);
175 mMainImageConsumer = new CpuConsumer(consumer, 1);
176 mMainImageConsumer->setFrameAvailableListener(this);
177 mMainImageConsumer->setName(String8("Camera3-HeicComposite-HevcInputYUVStream"));
178 }
179 mMainImageSurface = new Surface(producer);
180
181 res = mCodec->start();
182 if (res != OK) {
183 ALOGE("%s: Failed to start codec: %s (%d)", __FUNCTION__,
184 strerror(-res), res);
185 return res;
186 }
187
188 std::vector<int> sourceSurfaceId;
189 //Use YUV_888 format if framework tiling is needed.
190 int srcStreamFmt = mUseGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
191 HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
192 res = device->createStream(mMainImageSurface, width, height, srcStreamFmt, kHeifDataSpace,
193 rotation, id, physicalCameraId, sensorPixelModesUsed, &sourceSurfaceId,
194 camera3::CAMERA3_STREAM_SET_ID_INVALID, /*isShared*/false, /*isMultiResolution*/false,
195 /*consumerUsage*/0, ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
196 ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
197 OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
198 OutputConfiguration::MIRROR_MODE_AUTO,
199 colorSpace,
200 useReadoutTimestamp);
201 if (res == OK) {
202 mMainImageSurfaceId = sourceSurfaceId[0];
203 mMainImageStreamId = *id;
204 } else {
205 ALOGE("%s: Failed to create main image stream: %s (%d)", __FUNCTION__,
206 strerror(-res), res);
207 return res;
208 }
209
210 mOutputSurface = consumers[0];
211 res = registerCompositeStreamListener(mMainImageStreamId);
212 if (res != OK) {
213 ALOGE("%s: Failed to register HAL main image stream: %s (%d)", __FUNCTION__,
214 strerror(-res), res);
215 return res;
216 }
217
218 res = registerCompositeStreamListener(mAppSegmentStreamId);
219 if (res != OK) {
220 ALOGE("%s: Failed to register HAL app segment stream: %s (%d)", __FUNCTION__,
221 strerror(-res), res);
222 return res;
223 }
224
225 initCopyRowFunction(width);
226 return res;
227 }
228
deleteInternalStreams()229 status_t HeicCompositeStream::deleteInternalStreams() {
230 requestExit();
231 auto res = join();
232 if (res != OK) {
233 ALOGE("%s: Failed to join with the main processing thread: %s (%d)", __FUNCTION__,
234 strerror(-res), res);
235 }
236
237 deinitCodec();
238
239 if (mAppSegmentStreamId >= 0) {
240 // Camera devices may not be valid after switching to offline mode.
241 // In this case, all offline streams including internal composite streams
242 // are managed and released by the offline session.
243 sp<CameraDeviceBase> device = mDevice.promote();
244 if (device.get() != nullptr) {
245 res = device->deleteStream(mAppSegmentStreamId);
246 }
247
248 mAppSegmentStreamId = -1;
249 }
250
251 if (mOutputSurface != nullptr) {
252 mOutputSurface->disconnect(NATIVE_WINDOW_API_CAMERA);
253 mOutputSurface.clear();
254 }
255
256 sp<StatusTracker> statusTracker = mStatusTracker.promote();
257 if (statusTracker != nullptr && mStatusId != StatusTracker::NO_STATUS_ID) {
258 statusTracker->removeComponent(mStatusId);
259 mStatusId = StatusTracker::NO_STATUS_ID;
260 }
261
262 if (mPendingInputFrames.size() > 0) {
263 ALOGW("%s: mPendingInputFrames has %zu stale entries",
264 __FUNCTION__, mPendingInputFrames.size());
265 mPendingInputFrames.clear();
266 }
267
268 return res;
269 }
270
onBufferReleased(const BufferInfo & bufferInfo)271 void HeicCompositeStream::onBufferReleased(const BufferInfo& bufferInfo) {
272 Mutex::Autolock l(mMutex);
273
274 if (bufferInfo.mError) return;
275
276 if (bufferInfo.mStreamId == mMainImageStreamId) {
277 mMainImageFrameNumbers.push(bufferInfo.mFrameNumber);
278 mCodecOutputBufferFrameNumbers.push(bufferInfo.mFrameNumber);
279 ALOGV("%s: [%" PRId64 "]: Adding main image frame number (%zu frame numbers in total)",
280 __FUNCTION__, bufferInfo.mFrameNumber, mMainImageFrameNumbers.size());
281 } else if (bufferInfo.mStreamId == mAppSegmentStreamId) {
282 mAppSegmentFrameNumbers.push(bufferInfo.mFrameNumber);
283 ALOGV("%s: [%" PRId64 "]: Adding app segment frame number (%zu frame numbers in total)",
284 __FUNCTION__, bufferInfo.mFrameNumber, mAppSegmentFrameNumbers.size());
285 }
286 }
287
288 // We need to get the settings early to handle the case where the codec output
289 // arrives earlier than result metadata.
onBufferRequestForFrameNumber(uint64_t frameNumber,int streamId,const CameraMetadata & settings)290 void HeicCompositeStream::onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId,
291 const CameraMetadata& settings) {
292 ATRACE_ASYNC_BEGIN("HEIC capture", frameNumber);
293
294 Mutex::Autolock l(mMutex);
295 if (mErrorState || (streamId != getStreamId())) {
296 return;
297 }
298
299 mPendingCaptureResults.emplace(frameNumber, CameraMetadata());
300
301 camera_metadata_ro_entry entry;
302
303 int32_t orientation = 0;
304 entry = settings.find(ANDROID_JPEG_ORIENTATION);
305 if (entry.count == 1) {
306 orientation = entry.data.i32[0];
307 }
308
309 int32_t quality = kDefaultJpegQuality;
310 entry = settings.find(ANDROID_JPEG_QUALITY);
311 if (entry.count == 1) {
312 quality = entry.data.i32[0];
313 }
314
315 mSettingsByFrameNumber[frameNumber] = {orientation, quality};
316 }
317
onFrameAvailable(const BufferItem & item)318 void HeicCompositeStream::onFrameAvailable(const BufferItem& item) {
319 if (item.mDataSpace == static_cast<android_dataspace>(kAppSegmentDataSpace)) {
320 ALOGV("%s: JPEG APP segments buffer with ts: %" PRIu64 " ms. arrived!",
321 __func__, ns2ms(item.mTimestamp));
322
323 Mutex::Autolock l(mMutex);
324 if (!mErrorState) {
325 mInputAppSegmentBuffers.push_back(item.mTimestamp);
326 mInputReadyCondition.signal();
327 }
328 } else if (item.mDataSpace == kHeifDataSpace) {
329 ALOGV("%s: YUV_888 buffer with ts: %" PRIu64 " ms. arrived!",
330 __func__, ns2ms(item.mTimestamp));
331
332 Mutex::Autolock l(mMutex);
333 if (!mUseGrid) {
334 ALOGE("%s: YUV_888 internal stream is only supported for HEVC tiling",
335 __FUNCTION__);
336 return;
337 }
338 if (!mErrorState) {
339 mInputYuvBuffers.push_back(item.mTimestamp);
340 mInputReadyCondition.signal();
341 }
342 } else {
343 ALOGE("%s: Unexpected data space: 0x%x", __FUNCTION__, item.mDataSpace);
344 }
345 }
346
getCompositeStreamInfo(const OutputStreamInfo & streamInfo,const CameraMetadata & ch,std::vector<OutputStreamInfo> * compositeOutput)347 status_t HeicCompositeStream::getCompositeStreamInfo(const OutputStreamInfo &streamInfo,
348 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/) {
349 if (compositeOutput == nullptr) {
350 return BAD_VALUE;
351 }
352
353 compositeOutput->clear();
354
355 bool useGrid, useHeic;
356 bool isSizeSupported = isSizeSupportedByHeifEncoder(
357 streamInfo.width, streamInfo.height, &useHeic, &useGrid, nullptr);
358 if (!isSizeSupported) {
359 // Size is not supported by either encoder.
360 return OK;
361 }
362
363 compositeOutput->insert(compositeOutput->end(), 2, streamInfo);
364
365 // JPEG APPS segments Blob stream info
366 (*compositeOutput)[0].width = calcAppSegmentMaxSize(ch);
367 (*compositeOutput)[0].height = 1;
368 (*compositeOutput)[0].format = HAL_PIXEL_FORMAT_BLOB;
369 (*compositeOutput)[0].dataSpace = kAppSegmentDataSpace;
370 (*compositeOutput)[0].consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
371
372 // YUV/IMPLEMENTATION_DEFINED stream info
373 (*compositeOutput)[1].width = streamInfo.width;
374 (*compositeOutput)[1].height = streamInfo.height;
375 (*compositeOutput)[1].format = useGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
376 HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
377 (*compositeOutput)[1].dataSpace = kHeifDataSpace;
378 (*compositeOutput)[1].consumerUsage = useHeic ? GRALLOC_USAGE_HW_IMAGE_ENCODER :
379 useGrid ? GRALLOC_USAGE_SW_READ_OFTEN : GRALLOC_USAGE_HW_VIDEO_ENCODER;
380
381 return NO_ERROR;
382 }
383
isSizeSupportedByHeifEncoder(int32_t width,int32_t height,bool * useHeic,bool * useGrid,int64_t * stall,AString * hevcName)384 bool HeicCompositeStream::isSizeSupportedByHeifEncoder(int32_t width, int32_t height,
385 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName) {
386 static HeicEncoderInfoManager& heicManager = HeicEncoderInfoManager::getInstance();
387 return heicManager.isSizeSupported(width, height, useHeic, useGrid, stall, hevcName);
388 }
389
isInMemoryTempFileSupported()390 bool HeicCompositeStream::isInMemoryTempFileSupported() {
391 int memfd = syscall(__NR_memfd_create, "HEIF-try-memfd", MFD_CLOEXEC);
392 if (memfd == -1) {
393 if (errno != ENOSYS) {
394 ALOGE("%s: Failed to create tmpfs file. errno %d", __FUNCTION__, errno);
395 }
396 return false;
397 }
398 close(memfd);
399 return true;
400 }
401
onHeicOutputFrameAvailable(const CodecOutputBufferInfo & outputBufferInfo)402 void HeicCompositeStream::onHeicOutputFrameAvailable(
403 const CodecOutputBufferInfo& outputBufferInfo) {
404 Mutex::Autolock l(mMutex);
405
406 ALOGV("%s: index %d, offset %d, size %d, time %" PRId64 ", flags 0x%x",
407 __FUNCTION__, outputBufferInfo.index, outputBufferInfo.offset,
408 outputBufferInfo.size, outputBufferInfo.timeUs, outputBufferInfo.flags);
409
410 if (!mErrorState) {
411 if ((outputBufferInfo.size > 0) &&
412 ((outputBufferInfo.flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) == 0)) {
413 mCodecOutputBuffers.push_back(outputBufferInfo);
414 mInputReadyCondition.signal();
415 } else {
416 ALOGV("%s: Releasing output buffer: size %d flags: 0x%x ", __FUNCTION__,
417 outputBufferInfo.size, outputBufferInfo.flags);
418 mCodec->releaseOutputBuffer(outputBufferInfo.index);
419 }
420 } else {
421 mCodec->releaseOutputBuffer(outputBufferInfo.index);
422 }
423 }
424
onHeicInputFrameAvailable(int32_t index)425 void HeicCompositeStream::onHeicInputFrameAvailable(int32_t index) {
426 Mutex::Autolock l(mMutex);
427
428 if (!mUseGrid) {
429 ALOGE("%s: Codec YUV input mode must only be used for Hevc tiling mode", __FUNCTION__);
430 return;
431 }
432
433 mCodecInputBuffers.push_back(index);
434 mInputReadyCondition.signal();
435 }
436
onHeicFormatChanged(sp<AMessage> & newFormat)437 void HeicCompositeStream::onHeicFormatChanged(sp<AMessage>& newFormat) {
438 if (newFormat == nullptr) {
439 ALOGE("%s: newFormat must not be null!", __FUNCTION__);
440 return;
441 }
442
443 Mutex::Autolock l(mMutex);
444
445 AString mime;
446 AString mimeHeic(MIMETYPE_IMAGE_ANDROID_HEIC);
447 newFormat->findString(KEY_MIME, &mime);
448 if (mime != mimeHeic) {
449 // For HEVC codec, below keys need to be filled out or overwritten so that the
450 // muxer can handle them as HEIC output image.
451 newFormat->setString(KEY_MIME, mimeHeic);
452 newFormat->setInt32(KEY_WIDTH, mOutputWidth);
453 newFormat->setInt32(KEY_HEIGHT, mOutputHeight);
454 if (mUseGrid) {
455 newFormat->setInt32(KEY_TILE_WIDTH, mGridWidth);
456 newFormat->setInt32(KEY_TILE_HEIGHT, mGridHeight);
457 newFormat->setInt32(KEY_GRID_ROWS, mGridRows);
458 newFormat->setInt32(KEY_GRID_COLUMNS, mGridCols);
459 int32_t left, top, right, bottom;
460 if (newFormat->findRect("crop", &left, &top, &right, &bottom)) {
461 newFormat->setRect("crop", 0, 0, mOutputWidth - 1, mOutputHeight - 1);
462 }
463 }
464 }
465 newFormat->setInt32(KEY_IS_DEFAULT, 1 /*isPrimary*/);
466
467 int32_t gridRows, gridCols;
468 if (newFormat->findInt32(KEY_GRID_ROWS, &gridRows) &&
469 newFormat->findInt32(KEY_GRID_COLUMNS, &gridCols)) {
470 mNumOutputTiles = gridRows * gridCols;
471 } else {
472 mNumOutputTiles = 1;
473 }
474
475 mFormat = newFormat;
476
477 ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles);
478 mInputReadyCondition.signal();
479 }
480
onHeicCodecError()481 void HeicCompositeStream::onHeicCodecError() {
482 Mutex::Autolock l(mMutex);
483 mErrorState = true;
484 }
485
configureStream()486 status_t HeicCompositeStream::configureStream() {
487 if (isRunning()) {
488 // Processing thread is already running, nothing more to do.
489 return NO_ERROR;
490 }
491
492 if (mOutputSurface.get() == nullptr) {
493 ALOGE("%s: No valid output surface set!", __FUNCTION__);
494 return NO_INIT;
495 }
496
497 auto res = mOutputSurface->connect(NATIVE_WINDOW_API_CAMERA, mProducerListener);
498 if (res != OK) {
499 ALOGE("%s: Unable to connect to native window for stream %d",
500 __FUNCTION__, mMainImageStreamId);
501 return res;
502 }
503
504 if ((res = native_window_set_buffers_format(mOutputSurface.get(), HAL_PIXEL_FORMAT_BLOB))
505 != OK) {
506 ALOGE("%s: Unable to configure stream buffer format for stream %d", __FUNCTION__,
507 mMainImageStreamId);
508 return res;
509 }
510
511 ANativeWindow *anwConsumer = mOutputSurface.get();
512 int maxConsumerBuffers;
513 if ((res = anwConsumer->query(anwConsumer, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS,
514 &maxConsumerBuffers)) != OK) {
515 ALOGE("%s: Unable to query consumer undequeued"
516 " buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
517 return res;
518 }
519
520 // Cannot use SourceSurface buffer count since it could be codec's 512*512 tile
521 // buffer count.
522 if ((res = native_window_set_buffer_count(
523 anwConsumer, kMaxOutputSurfaceProducerCount + maxConsumerBuffers)) != OK) {
524 ALOGE("%s: Unable to set buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
525 return res;
526 }
527
528 if ((res = native_window_set_buffers_dimensions(anwConsumer, mMaxHeicBufferSize, 1)) != OK) {
529 ALOGE("%s: Unable to set buffer dimension %zu x 1 for stream %d: %s (%d)",
530 __FUNCTION__, mMaxHeicBufferSize, mMainImageStreamId, strerror(-res), res);
531 return res;
532 }
533
534 sp<camera3::StatusTracker> statusTracker = mStatusTracker.promote();
535 if (statusTracker != nullptr) {
536 std::string name = std::string("HeicStream ") + std::to_string(getStreamId());
537 mStatusId = statusTracker->addComponent(name);
538 }
539
540 run("HeicCompositeStreamProc");
541
542 return NO_ERROR;
543 }
544
insertGbp(SurfaceMap * outSurfaceMap,Vector<int32_t> * outputStreamIds,int32_t * currentStreamId)545 status_t HeicCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap,
546 Vector<int32_t>* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) {
547 if (outSurfaceMap->find(mAppSegmentStreamId) == outSurfaceMap->end()) {
548 outputStreamIds->push_back(mAppSegmentStreamId);
549 }
550 (*outSurfaceMap)[mAppSegmentStreamId].push_back(mAppSegmentSurfaceId);
551
552 if (outSurfaceMap->find(mMainImageStreamId) == outSurfaceMap->end()) {
553 outputStreamIds->push_back(mMainImageStreamId);
554 }
555 (*outSurfaceMap)[mMainImageStreamId].push_back(mMainImageSurfaceId);
556
557 if (currentStreamId != nullptr) {
558 *currentStreamId = mMainImageStreamId;
559 }
560
561 return NO_ERROR;
562 }
563
insertCompositeStreamIds(std::vector<int32_t> * compositeStreamIds)564 status_t HeicCompositeStream::insertCompositeStreamIds(
565 std::vector<int32_t>* compositeStreamIds /*out*/) {
566 if (compositeStreamIds == nullptr) {
567 return BAD_VALUE;
568 }
569
570 compositeStreamIds->push_back(mAppSegmentStreamId);
571 compositeStreamIds->push_back(mMainImageStreamId);
572
573 return OK;
574 }
575
onShutter(const CaptureResultExtras & resultExtras,nsecs_t timestamp)576 void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) {
577 Mutex::Autolock l(mMutex);
578 if (mErrorState) {
579 return;
580 }
581
582 if (mSettingsByFrameNumber.find(resultExtras.frameNumber) != mSettingsByFrameNumber.end()) {
583 ALOGV("%s: [%" PRId64 "]: timestamp %" PRId64 ", requestId %d", __FUNCTION__,
584 resultExtras.frameNumber, timestamp, resultExtras.requestId);
585 mSettingsByFrameNumber[resultExtras.frameNumber].shutterNotified = true;
586 mSettingsByFrameNumber[resultExtras.frameNumber].timestamp = timestamp;
587 mSettingsByFrameNumber[resultExtras.frameNumber].requestId = resultExtras.requestId;
588 mInputReadyCondition.signal();
589 }
590 }
591
compilePendingInputLocked()592 void HeicCompositeStream::compilePendingInputLocked() {
593 auto i = mSettingsByFrameNumber.begin();
594 while (i != mSettingsByFrameNumber.end()) {
595 if (i->second.shutterNotified) {
596 mPendingInputFrames[i->first].orientation = i->second.orientation;
597 mPendingInputFrames[i->first].quality = i->second.quality;
598 mPendingInputFrames[i->first].timestamp = i->second.timestamp;
599 mPendingInputFrames[i->first].requestId = i->second.requestId;
600 ALOGV("%s: [%" PRId64 "]: timestamp is %" PRId64, __FUNCTION__,
601 i->first, i->second.timestamp);
602 i = mSettingsByFrameNumber.erase(i);
603
604 // Set encoder quality if no inflight encoding
605 if (mPendingInputFrames.size() == 1) {
606 sp<StatusTracker> statusTracker = mStatusTracker.promote();
607 if (statusTracker != nullptr) {
608 statusTracker->markComponentActive(mStatusId);
609 ALOGV("%s: Mark component as active", __FUNCTION__);
610 }
611
612 int32_t newQuality = mPendingInputFrames.begin()->second.quality;
613 updateCodecQualityLocked(newQuality);
614 }
615 } else {
616 i++;
617 }
618 }
619
620 while (!mInputAppSegmentBuffers.empty() && mAppSegmentFrameNumbers.size() > 0) {
621 CpuConsumer::LockedBuffer imgBuffer;
622 auto it = mInputAppSegmentBuffers.begin();
623 auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer);
624 if (res == NOT_ENOUGH_DATA) {
625 // Can not lock any more buffers.
626 break;
627 } else if ((res != OK) || (*it != imgBuffer.timestamp)) {
628 if (res != OK) {
629 ALOGE("%s: Error locking JPEG_APP_SEGMENTS image buffer: %s (%d)", __FUNCTION__,
630 strerror(-res), res);
631 } else {
632 ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64
633 " received buffer with time stamp: %" PRId64, __FUNCTION__,
634 *it, imgBuffer.timestamp);
635 mAppSegmentConsumer->unlockBuffer(imgBuffer);
636 }
637 mPendingInputFrames[*it].error = true;
638 mInputAppSegmentBuffers.erase(it);
639 continue;
640 }
641
642 if (mPendingInputFrames.find(mAppSegmentFrameNumbers.front()) == mPendingInputFrames.end()) {
643 ALOGE("%s: mPendingInputFrames doesn't contain frameNumber %" PRId64, __FUNCTION__,
644 mAppSegmentFrameNumbers.front());
645 mInputAppSegmentBuffers.erase(it);
646 mAppSegmentFrameNumbers.pop();
647 continue;
648 }
649
650 int64_t frameNumber = mAppSegmentFrameNumbers.front();
651 // If mPendingInputFrames doesn't contain the expected frame number, the captured
652 // input app segment frame must have been dropped via a buffer error. Simply
653 // return the buffer to the buffer queue.
654 if ((mPendingInputFrames.find(frameNumber) == mPendingInputFrames.end()) ||
655 (mPendingInputFrames[frameNumber].error)) {
656 mAppSegmentConsumer->unlockBuffer(imgBuffer);
657 } else {
658 mPendingInputFrames[frameNumber].appSegmentBuffer = imgBuffer;
659 }
660 mInputAppSegmentBuffers.erase(it);
661 mAppSegmentFrameNumbers.pop();
662 }
663
664 while (!mInputYuvBuffers.empty() && !mYuvBufferAcquired && mMainImageFrameNumbers.size() > 0) {
665 CpuConsumer::LockedBuffer imgBuffer;
666 auto it = mInputYuvBuffers.begin();
667 auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer);
668 if (res == NOT_ENOUGH_DATA) {
669 // Can not lock any more buffers.
670 break;
671 } else if (res != OK) {
672 ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__,
673 strerror(-res), res);
674 mPendingInputFrames[*it].error = true;
675 mInputYuvBuffers.erase(it);
676 continue;
677 } else if (*it != imgBuffer.timestamp) {
678 ALOGW("%s: Expecting YUV_888 buffer with time stamp: %" PRId64 " received buffer with "
679 "time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp);
680 mPendingInputFrames[*it].error = true;
681 mInputYuvBuffers.erase(it);
682 continue;
683 }
684
685 if (mPendingInputFrames.find(mMainImageFrameNumbers.front()) == mPendingInputFrames.end()) {
686 ALOGE("%s: mPendingInputFrames doesn't contain frameNumber %" PRId64, __FUNCTION__,
687 mMainImageFrameNumbers.front());
688 mInputYuvBuffers.erase(it);
689 mMainImageFrameNumbers.pop();
690 continue;
691 }
692
693 int64_t frameNumber = mMainImageFrameNumbers.front();
694 // If mPendingInputFrames doesn't contain the expected frame number, the captured
695 // input main image must have been dropped via a buffer error. Simply
696 // return the buffer to the buffer queue.
697 if ((mPendingInputFrames.find(frameNumber) == mPendingInputFrames.end()) ||
698 (mPendingInputFrames[frameNumber].error)) {
699 mMainImageConsumer->unlockBuffer(imgBuffer);
700 } else {
701 mPendingInputFrames[frameNumber].yuvBuffer = imgBuffer;
702 mYuvBufferAcquired = true;
703 }
704 mInputYuvBuffers.erase(it);
705 mMainImageFrameNumbers.pop();
706 }
707
708 while (!mCodecOutputBuffers.empty()) {
709 auto it = mCodecOutputBuffers.begin();
710 // Assume encoder input to output is FIFO, use a queue to look up
711 // frameNumber when handling codec outputs.
712 int64_t bufferFrameNumber = -1;
713 if (mCodecOutputBufferFrameNumbers.empty()) {
714 ALOGV("%s: Failed to find buffer frameNumber for codec output buffer!", __FUNCTION__);
715 break;
716 } else {
717 // Direct mapping between camera frame number and codec timestamp (in us).
718 bufferFrameNumber = mCodecOutputBufferFrameNumbers.front();
719 mCodecOutputCounter++;
720 if (mCodecOutputCounter == mNumOutputTiles) {
721 mCodecOutputBufferFrameNumbers.pop();
722 mCodecOutputCounter = 0;
723 }
724
725 mPendingInputFrames[bufferFrameNumber].codecOutputBuffers.push_back(*it);
726 ALOGV("%s: [%" PRId64 "]: Pushing codecOutputBuffers (frameNumber %" PRId64 ")",
727 __FUNCTION__, bufferFrameNumber, it->timeUs);
728 }
729 mCodecOutputBuffers.erase(it);
730 }
731
732 while (!mCaptureResults.empty()) {
733 auto it = mCaptureResults.begin();
734 // Negative frame number indicates that something went wrong during the capture result
735 // collection process.
736 int64_t frameNumber = std::get<0>(it->second);
737 if (it->first >= 0 &&
738 mPendingInputFrames.find(frameNumber) != mPendingInputFrames.end()) {
739 if (mPendingInputFrames[frameNumber].timestamp == it->first) {
740 mPendingInputFrames[frameNumber].result =
741 std::make_unique<CameraMetadata>(std::get<1>(it->second));
742 } else {
743 ALOGE("%s: Capture result frameNumber/timestamp mapping changed between "
744 "shutter and capture result! before: %" PRId64 ", after: %" PRId64,
745 __FUNCTION__, mPendingInputFrames[frameNumber].timestamp,
746 it->first);
747 }
748 }
749 mCaptureResults.erase(it);
750 }
751
752 // mErrorFrameNumbers stores frame number of dropped buffers.
753 auto it = mErrorFrameNumbers.begin();
754 while (it != mErrorFrameNumbers.end()) {
755 if (mPendingInputFrames.find(*it) != mPendingInputFrames.end()) {
756 mPendingInputFrames[*it].error = true;
757 } else {
758 //Error callback is guaranteed to arrive after shutter notify, which
759 //results in mPendingInputFrames being populated.
760 ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__,
761 *it);
762 }
763 it = mErrorFrameNumbers.erase(it);
764 }
765
766 // mExifErrorFrameNumbers stores the frame number of dropped APP_SEGMENT buffers
767 it = mExifErrorFrameNumbers.begin();
768 while (it != mExifErrorFrameNumbers.end()) {
769 if (mPendingInputFrames.find(*it) != mPendingInputFrames.end()) {
770 mPendingInputFrames[*it].exifError = true;
771 }
772 it = mExifErrorFrameNumbers.erase(it);
773 }
774
775 // Distribute codec input buffers to be filled out from YUV output
776 for (auto it = mPendingInputFrames.begin();
777 it != mPendingInputFrames.end() && mCodecInputBuffers.size() > 0; it++) {
778 InputFrame& inputFrame(it->second);
779 if (inputFrame.codecInputCounter < mGridRows * mGridCols) {
780 // Available input tiles that are required for the current input
781 // image.
782 size_t newInputTiles = std::min(mCodecInputBuffers.size(),
783 mGridRows * mGridCols - inputFrame.codecInputCounter);
784 for (size_t i = 0; i < newInputTiles; i++) {
785 CodecInputBufferInfo inputInfo =
786 { mCodecInputBuffers[0], mGridTimestampUs++, inputFrame.codecInputCounter };
787 inputFrame.codecInputBuffers.push_back(inputInfo);
788
789 mCodecInputBuffers.erase(mCodecInputBuffers.begin());
790 inputFrame.codecInputCounter++;
791 }
792 break;
793 }
794 }
795 }
796
getNextReadyInputLocked(int64_t * frameNumber)797 bool HeicCompositeStream::getNextReadyInputLocked(int64_t *frameNumber /*out*/) {
798 if (frameNumber == nullptr) {
799 return false;
800 }
801
802 bool newInputAvailable = false;
803 for (auto& it : mPendingInputFrames) {
804 // New input is considered to be available only if:
805 // 1. input buffers are ready, or
806 // 2. App segment and muxer is created, or
807 // 3. A codec output tile is ready, and an output buffer is available.
808 // This makes sure that muxer gets created only when an output tile is
809 // generated, because right now we only handle 1 HEIC output buffer at a
810 // time (max dequeued buffer count is 1).
811 bool appSegmentReady =
812 (it.second.appSegmentBuffer.data != nullptr || it.second.exifError) &&
813 !it.second.appSegmentWritten && it.second.result != nullptr &&
814 it.second.muxer != nullptr;
815 bool codecOutputReady = !it.second.codecOutputBuffers.empty();
816 bool codecInputReady = (it.second.yuvBuffer.data != nullptr) &&
817 (!it.second.codecInputBuffers.empty());
818 bool hasOutputBuffer = it.second.muxer != nullptr ||
819 (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount);
820 if ((!it.second.error) &&
821 (appSegmentReady || (codecOutputReady && hasOutputBuffer) || codecInputReady)) {
822 *frameNumber = it.first;
823 if (it.second.format == nullptr && mFormat != nullptr) {
824 it.second.format = mFormat->dup();
825 }
826 newInputAvailable = true;
827 break;
828 }
829 }
830
831 return newInputAvailable;
832 }
833
getNextFailingInputLocked()834 int64_t HeicCompositeStream::getNextFailingInputLocked() {
835 int64_t res = -1;
836
837 for (const auto& it : mPendingInputFrames) {
838 if (it.second.error) {
839 res = it.first;
840 break;
841 }
842 }
843
844 return res;
845 }
846
processInputFrame(int64_t frameNumber,InputFrame & inputFrame)847 status_t HeicCompositeStream::processInputFrame(int64_t frameNumber,
848 InputFrame &inputFrame) {
849 ATRACE_CALL();
850 status_t res = OK;
851
852 bool appSegmentReady =
853 (inputFrame.appSegmentBuffer.data != nullptr || inputFrame.exifError) &&
854 !inputFrame.appSegmentWritten && inputFrame.result != nullptr &&
855 inputFrame.muxer != nullptr;
856 bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0;
857 bool codecInputReady = inputFrame.yuvBuffer.data != nullptr &&
858 !inputFrame.codecInputBuffers.empty();
859 bool hasOutputBuffer = inputFrame.muxer != nullptr ||
860 (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount);
861
862 ALOGV("%s: [%" PRId64 "]: appSegmentReady %d, codecOutputReady %d, codecInputReady %d,"
863 " dequeuedOutputBuffer %d, timestamp %" PRId64, __FUNCTION__, frameNumber,
864 appSegmentReady, codecOutputReady, codecInputReady, mDequeuedOutputBufferCnt,
865 inputFrame.timestamp);
866
867 // Handle inputs for Hevc tiling
868 if (codecInputReady) {
869 res = processCodecInputFrame(inputFrame);
870 if (res != OK) {
871 ALOGE("%s: Failed to process codec input frame: %s (%d)", __FUNCTION__,
872 strerror(-res), res);
873 return res;
874 }
875 }
876
877 if (!(codecOutputReady && hasOutputBuffer) && !appSegmentReady) {
878 return OK;
879 }
880
881 // Initialize and start muxer if not yet done so. In this case,
882 // codecOutputReady must be true. Otherwise, appSegmentReady is guaranteed
883 // to be false, and the function must have returned early.
884 if (inputFrame.muxer == nullptr) {
885 res = startMuxerForInputFrame(frameNumber, inputFrame);
886 if (res != OK) {
887 ALOGE("%s: Failed to create and start muxer: %s (%d)", __FUNCTION__,
888 strerror(-res), res);
889 return res;
890 }
891 }
892
893 // Write JPEG APP segments data to the muxer.
894 if (appSegmentReady) {
895 res = processAppSegment(frameNumber, inputFrame);
896 if (res != OK) {
897 ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__,
898 strerror(-res), res);
899 return res;
900 }
901 }
902
903 // Write media codec bitstream buffers to muxer.
904 while (!inputFrame.codecOutputBuffers.empty()) {
905 res = processOneCodecOutputFrame(frameNumber, inputFrame);
906 if (res != OK) {
907 ALOGE("%s: Failed to process codec output frame: %s (%d)", __FUNCTION__,
908 strerror(-res), res);
909 return res;
910 }
911 }
912
913 if (inputFrame.pendingOutputTiles == 0) {
914 if (inputFrame.appSegmentWritten) {
915 res = processCompletedInputFrame(frameNumber, inputFrame);
916 if (res != OK) {
917 ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__,
918 strerror(-res), res);
919 return res;
920 }
921 }
922 }
923
924 return res;
925 }
926
startMuxerForInputFrame(int64_t frameNumber,InputFrame & inputFrame)927 status_t HeicCompositeStream::startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame) {
928 sp<ANativeWindow> outputANW = mOutputSurface;
929
930 auto res = outputANW->dequeueBuffer(mOutputSurface.get(), &inputFrame.anb, &inputFrame.fenceFd);
931 if (res != OK) {
932 ALOGE("%s: Error retrieving output buffer: %s (%d)", __FUNCTION__, strerror(-res),
933 res);
934 return res;
935 }
936 mDequeuedOutputBufferCnt++;
937
938 // Combine current thread id, stream id and timestamp to uniquely identify image.
939 std::ostringstream tempOutputFile;
940 tempOutputFile << "HEIF-" << pthread_self() << "-"
941 << getStreamId() << "-" << frameNumber;
942 inputFrame.fileFd = syscall(__NR_memfd_create, tempOutputFile.str().c_str(), MFD_CLOEXEC);
943 if (inputFrame.fileFd < 0) {
944 ALOGE("%s: Failed to create file %s. Error no is %d", __FUNCTION__,
945 tempOutputFile.str().c_str(), errno);
946 return NO_INIT;
947 }
948 inputFrame.muxer = MediaMuxer::create(inputFrame.fileFd, MediaMuxer::OUTPUT_FORMAT_HEIF);
949 if (inputFrame.muxer == nullptr) {
950 ALOGE("%s: Failed to create MediaMuxer for file fd %d",
951 __FUNCTION__, inputFrame.fileFd);
952 return NO_INIT;
953 }
954
955 res = inputFrame.muxer->setOrientationHint(inputFrame.orientation);
956 if (res != OK) {
957 ALOGE("%s: Failed to setOrientationHint: %s (%d)", __FUNCTION__,
958 strerror(-res), res);
959 return res;
960 }
961
962 ssize_t trackId = inputFrame.muxer->addTrack(inputFrame.format);
963 if (trackId < 0) {
964 ALOGE("%s: Failed to addTrack to the muxer: %zd", __FUNCTION__, trackId);
965 return NO_INIT;
966 }
967
968 inputFrame.trackIndex = trackId;
969 inputFrame.pendingOutputTiles = mNumOutputTiles;
970
971 res = inputFrame.muxer->start();
972 if (res != OK) {
973 ALOGE("%s: Failed to start MediaMuxer: %s (%d)",
974 __FUNCTION__, strerror(-res), res);
975 return res;
976 }
977
978 ALOGV("%s: [%" PRId64 "]: Muxer started for inputFrame", __FUNCTION__,
979 frameNumber);
980 return OK;
981 }
982
processAppSegment(int64_t frameNumber,InputFrame & inputFrame)983 status_t HeicCompositeStream::processAppSegment(int64_t frameNumber, InputFrame &inputFrame) {
984 size_t app1Size = 0;
985 size_t appSegmentSize = 0;
986 if (!inputFrame.exifError) {
987 appSegmentSize = findAppSegmentsSize(inputFrame.appSegmentBuffer.data,
988 inputFrame.appSegmentBuffer.width * inputFrame.appSegmentBuffer.height,
989 &app1Size);
990 if (appSegmentSize == 0) {
991 ALOGE("%s: Failed to find JPEG APP segment size", __FUNCTION__);
992 return NO_INIT;
993 }
994 }
995
996 std::unique_ptr<ExifUtils> exifUtils(ExifUtils::create());
997 auto exifRes = inputFrame.exifError ?
998 exifUtils->initializeEmpty() :
999 exifUtils->initialize(inputFrame.appSegmentBuffer.data, app1Size);
1000 if (!exifRes) {
1001 ALOGE("%s: Failed to initialize ExifUtils object!", __FUNCTION__);
1002 return BAD_VALUE;
1003 }
1004 exifRes = exifUtils->setFromMetadata(*inputFrame.result, mStaticInfo,
1005 mOutputWidth, mOutputHeight);
1006 if (!exifRes) {
1007 ALOGE("%s: Failed to set Exif tags using metadata and main image sizes", __FUNCTION__);
1008 return BAD_VALUE;
1009 }
1010 exifRes = exifUtils->setOrientation(inputFrame.orientation);
1011 if (!exifRes) {
1012 ALOGE("%s: ExifUtils failed to set orientation", __FUNCTION__);
1013 return BAD_VALUE;
1014 }
1015 exifRes = exifUtils->generateApp1();
1016 if (!exifRes) {
1017 ALOGE("%s: ExifUtils failed to generate APP1 segment", __FUNCTION__);
1018 return BAD_VALUE;
1019 }
1020
1021 unsigned int newApp1Length = exifUtils->getApp1Length();
1022 const uint8_t *newApp1Segment = exifUtils->getApp1Buffer();
1023
1024 //Assemble the APP1 marker buffer required by MediaCodec
1025 uint8_t kExifApp1Marker[] = {'E', 'x', 'i', 'f', 0xFF, 0xE1, 0x00, 0x00};
1026 kExifApp1Marker[6] = static_cast<uint8_t>(newApp1Length >> 8);
1027 kExifApp1Marker[7] = static_cast<uint8_t>(newApp1Length & 0xFF);
1028 size_t appSegmentBufferSize = sizeof(kExifApp1Marker) +
1029 appSegmentSize - app1Size + newApp1Length;
1030 uint8_t* appSegmentBuffer = new uint8_t[appSegmentBufferSize];
1031 memcpy(appSegmentBuffer, kExifApp1Marker, sizeof(kExifApp1Marker));
1032 memcpy(appSegmentBuffer + sizeof(kExifApp1Marker), newApp1Segment, newApp1Length);
1033 if (appSegmentSize - app1Size > 0) {
1034 memcpy(appSegmentBuffer + sizeof(kExifApp1Marker) + newApp1Length,
1035 inputFrame.appSegmentBuffer.data + app1Size, appSegmentSize - app1Size);
1036 }
1037
1038 sp<ABuffer> aBuffer = new ABuffer(appSegmentBuffer, appSegmentBufferSize);
1039 auto res = inputFrame.muxer->writeSampleData(aBuffer, inputFrame.trackIndex,
1040 inputFrame.timestamp, MediaCodec::BUFFER_FLAG_MUXER_DATA);
1041 delete[] appSegmentBuffer;
1042
1043 if (res != OK) {
1044 ALOGE("%s: Failed to write JPEG APP segments to muxer: %s (%d)",
1045 __FUNCTION__, strerror(-res), res);
1046 return res;
1047 }
1048
1049 ALOGV("%s: [%" PRId64 "]: appSegmentSize is %zu, width %d, height %d, app1Size %zu",
1050 __FUNCTION__, frameNumber, appSegmentSize, inputFrame.appSegmentBuffer.width,
1051 inputFrame.appSegmentBuffer.height, app1Size);
1052
1053 inputFrame.appSegmentWritten = true;
1054 // Release the buffer now so any pending input app segments can be processed
1055 mAppSegmentConsumer->unlockBuffer(inputFrame.appSegmentBuffer);
1056 inputFrame.appSegmentBuffer.data = nullptr;
1057 inputFrame.exifError = false;
1058
1059 return OK;
1060 }
1061
processCodecInputFrame(InputFrame & inputFrame)1062 status_t HeicCompositeStream::processCodecInputFrame(InputFrame &inputFrame) {
1063 for (auto& inputBuffer : inputFrame.codecInputBuffers) {
1064 sp<MediaCodecBuffer> buffer;
1065 auto res = mCodec->getInputBuffer(inputBuffer.index, &buffer);
1066 if (res != OK) {
1067 ALOGE("%s: Error getting codec input buffer: %s (%d)", __FUNCTION__,
1068 strerror(-res), res);
1069 return res;
1070 }
1071
1072 // Copy one tile from source to destination.
1073 size_t tileX = inputBuffer.tileIndex % mGridCols;
1074 size_t tileY = inputBuffer.tileIndex / mGridCols;
1075 size_t top = mGridHeight * tileY;
1076 size_t left = mGridWidth * tileX;
1077 size_t width = (tileX == static_cast<size_t>(mGridCols) - 1) ?
1078 mOutputWidth - tileX * mGridWidth : mGridWidth;
1079 size_t height = (tileY == static_cast<size_t>(mGridRows) - 1) ?
1080 mOutputHeight - tileY * mGridHeight : mGridHeight;
1081 ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu,"
1082 " timeUs %" PRId64, __FUNCTION__, tileX, tileY, top, left, width, height,
1083 inputBuffer.timeUs);
1084
1085 res = copyOneYuvTile(buffer, inputFrame.yuvBuffer, top, left, width, height);
1086 if (res != OK) {
1087 ALOGE("%s: Failed to copy YUV tile %s (%d)", __FUNCTION__,
1088 strerror(-res), res);
1089 return res;
1090 }
1091
1092 res = mCodec->queueInputBuffer(inputBuffer.index, 0, buffer->capacity(),
1093 inputBuffer.timeUs, 0, nullptr /*errorDetailMsg*/);
1094 if (res != OK) {
1095 ALOGE("%s: Failed to queueInputBuffer to Codec: %s (%d)",
1096 __FUNCTION__, strerror(-res), res);
1097 return res;
1098 }
1099 }
1100
1101 inputFrame.codecInputBuffers.clear();
1102 return OK;
1103 }
1104
processOneCodecOutputFrame(int64_t frameNumber,InputFrame & inputFrame)1105 status_t HeicCompositeStream::processOneCodecOutputFrame(int64_t frameNumber,
1106 InputFrame &inputFrame) {
1107 auto it = inputFrame.codecOutputBuffers.begin();
1108 sp<MediaCodecBuffer> buffer;
1109 status_t res = mCodec->getOutputBuffer(it->index, &buffer);
1110 if (res != OK) {
1111 ALOGE("%s: Error getting Heic codec output buffer at index %d: %s (%d)",
1112 __FUNCTION__, it->index, strerror(-res), res);
1113 return res;
1114 }
1115 if (buffer == nullptr) {
1116 ALOGE("%s: Invalid Heic codec output buffer at index %d",
1117 __FUNCTION__, it->index);
1118 return BAD_VALUE;
1119 }
1120
1121 sp<ABuffer> aBuffer = new ABuffer(buffer->data(), buffer->size());
1122 res = inputFrame.muxer->writeSampleData(
1123 aBuffer, inputFrame.trackIndex, inputFrame.timestamp, 0 /*flags*/);
1124 if (res != OK) {
1125 ALOGE("%s: Failed to write buffer index %d to muxer: %s (%d)",
1126 __FUNCTION__, it->index, strerror(-res), res);
1127 return res;
1128 }
1129
1130 mCodec->releaseOutputBuffer(it->index);
1131 if (inputFrame.pendingOutputTiles == 0) {
1132 ALOGW("%s: Codec generated more tiles than expected!", __FUNCTION__);
1133 } else {
1134 inputFrame.pendingOutputTiles--;
1135 }
1136
1137 inputFrame.codecOutputBuffers.erase(inputFrame.codecOutputBuffers.begin());
1138
1139 ALOGV("%s: [%" PRId64 "]: Output buffer index %d",
1140 __FUNCTION__, frameNumber, it->index);
1141 return OK;
1142 }
1143
processCompletedInputFrame(int64_t frameNumber,InputFrame & inputFrame)1144 status_t HeicCompositeStream::processCompletedInputFrame(int64_t frameNumber,
1145 InputFrame &inputFrame) {
1146 sp<ANativeWindow> outputANW = mOutputSurface;
1147 inputFrame.muxer->stop();
1148
1149 // Copy the content of the file to memory.
1150 sp<GraphicBuffer> gb = GraphicBuffer::from(inputFrame.anb);
1151 void* dstBuffer;
1152 GraphicBufferLocker gbLocker(gb);
1153 auto res = gbLocker.lockAsync(&dstBuffer, inputFrame.fenceFd);
1154 if (res != OK) {
1155 ALOGE("%s: Error trying to lock output buffer fence: %s (%d)", __FUNCTION__,
1156 strerror(-res), res);
1157 return res;
1158 }
1159
1160 off_t fSize = lseek(inputFrame.fileFd, 0, SEEK_END);
1161 if (static_cast<size_t>(fSize) > mMaxHeicBufferSize - sizeof(CameraBlob)) {
1162 ALOGE("%s: Error: MediaMuxer output size %ld is larger than buffer sizer %zu",
1163 __FUNCTION__, fSize, mMaxHeicBufferSize - sizeof(CameraBlob));
1164 return BAD_VALUE;
1165 }
1166
1167 lseek(inputFrame.fileFd, 0, SEEK_SET);
1168 ssize_t bytesRead = read(inputFrame.fileFd, dstBuffer, fSize);
1169 if (bytesRead < fSize) {
1170 ALOGE("%s: Only %zd of %ld bytes read", __FUNCTION__, bytesRead, fSize);
1171 return BAD_VALUE;
1172 }
1173
1174 close(inputFrame.fileFd);
1175 inputFrame.fileFd = -1;
1176
1177 // Fill in HEIC header
1178 // Must be in sync with CAMERA3_HEIC_BLOB_ID in android_media_Utils.cpp
1179 uint8_t *header = static_cast<uint8_t*>(dstBuffer) + mMaxHeicBufferSize - sizeof(CameraBlob);
1180 CameraBlob blobHeader = {
1181 .blobId = static_cast<CameraBlobId>(0x00FE),
1182 .blobSizeBytes = static_cast<int32_t>(fSize)
1183 };
1184 memcpy(header, &blobHeader, sizeof(CameraBlob));
1185
1186 res = native_window_set_buffers_timestamp(mOutputSurface.get(), inputFrame.timestamp);
1187 if (res != OK) {
1188 ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
1189 __FUNCTION__, getStreamId(), strerror(-res), res);
1190 return res;
1191 }
1192
1193 res = outputANW->queueBuffer(mOutputSurface.get(), inputFrame.anb, /*fence*/ -1);
1194 if (res != OK) {
1195 ALOGE("%s: Failed to queueBuffer to Heic stream: %s (%d)", __FUNCTION__,
1196 strerror(-res), res);
1197 return res;
1198 }
1199 inputFrame.anb = nullptr;
1200 mDequeuedOutputBufferCnt--;
1201
1202 ALOGV("%s: [%" PRId64 "]", __FUNCTION__, frameNumber);
1203 ATRACE_ASYNC_END("HEIC capture", frameNumber);
1204 return OK;
1205 }
1206
1207
releaseInputFrameLocked(int64_t frameNumber,InputFrame * inputFrame)1208 void HeicCompositeStream::releaseInputFrameLocked(int64_t frameNumber,
1209 InputFrame *inputFrame /*out*/) {
1210 if (inputFrame == nullptr) {
1211 return;
1212 }
1213
1214 if (inputFrame->appSegmentBuffer.data != nullptr) {
1215 mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer);
1216 inputFrame->appSegmentBuffer.data = nullptr;
1217 }
1218
1219 while (!inputFrame->codecOutputBuffers.empty()) {
1220 auto it = inputFrame->codecOutputBuffers.begin();
1221 ALOGV("%s: releaseOutputBuffer index %d", __FUNCTION__, it->index);
1222 mCodec->releaseOutputBuffer(it->index);
1223 inputFrame->codecOutputBuffers.erase(it);
1224 }
1225
1226 if (inputFrame->yuvBuffer.data != nullptr) {
1227 mMainImageConsumer->unlockBuffer(inputFrame->yuvBuffer);
1228 inputFrame->yuvBuffer.data = nullptr;
1229 mYuvBufferAcquired = false;
1230 }
1231
1232 while (!inputFrame->codecInputBuffers.empty()) {
1233 auto it = inputFrame->codecInputBuffers.begin();
1234 inputFrame->codecInputBuffers.erase(it);
1235 }
1236
1237 if (inputFrame->error || mErrorState) {
1238 ALOGV("%s: notifyError called for frameNumber %" PRId64, __FUNCTION__, frameNumber);
1239 notifyError(frameNumber, inputFrame->requestId);
1240 }
1241
1242 if (inputFrame->fileFd >= 0) {
1243 close(inputFrame->fileFd);
1244 inputFrame->fileFd = -1;
1245 }
1246
1247 if (inputFrame->anb != nullptr) {
1248 sp<ANativeWindow> outputANW = mOutputSurface;
1249 outputANW->cancelBuffer(mOutputSurface.get(), inputFrame->anb, /*fence*/ -1);
1250 inputFrame->anb = nullptr;
1251
1252 mDequeuedOutputBufferCnt--;
1253 }
1254 }
1255
releaseInputFramesLocked()1256 void HeicCompositeStream::releaseInputFramesLocked() {
1257 auto it = mPendingInputFrames.begin();
1258 bool inputFrameDone = false;
1259 while (it != mPendingInputFrames.end()) {
1260 auto& inputFrame = it->second;
1261 if (inputFrame.error ||
1262 (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0)) {
1263 releaseInputFrameLocked(it->first, &inputFrame);
1264 it = mPendingInputFrames.erase(it);
1265 inputFrameDone = true;
1266 } else {
1267 it++;
1268 }
1269 }
1270
1271 // Update codec quality based on first upcoming input frame.
1272 // Note that when encoding is in surface mode, currently there is no
1273 // way for camera service to synchronize quality setting on a per-frame
1274 // basis: we don't get notification when codec is ready to consume a new
1275 // input frame. So we update codec quality on a best-effort basis.
1276 if (inputFrameDone) {
1277 auto firstPendingFrame = mPendingInputFrames.begin();
1278 if (firstPendingFrame != mPendingInputFrames.end()) {
1279 updateCodecQualityLocked(firstPendingFrame->second.quality);
1280 } else {
1281 markTrackerIdle();
1282 }
1283 }
1284 }
1285
initializeCodec(uint32_t width,uint32_t height,const sp<CameraDeviceBase> & cameraDevice)1286 status_t HeicCompositeStream::initializeCodec(uint32_t width, uint32_t height,
1287 const sp<CameraDeviceBase>& cameraDevice) {
1288 ALOGV("%s", __FUNCTION__);
1289
1290 bool useGrid = false;
1291 AString hevcName;
1292 bool isSizeSupported = isSizeSupportedByHeifEncoder(width, height,
1293 &mUseHeic, &useGrid, nullptr, &hevcName);
1294 if (!isSizeSupported) {
1295 ALOGE("%s: Encoder doesnt' support size %u x %u!",
1296 __FUNCTION__, width, height);
1297 return BAD_VALUE;
1298 }
1299
1300 // Create Looper for MediaCodec.
1301 auto desiredMime = mUseHeic ? MIMETYPE_IMAGE_ANDROID_HEIC : MIMETYPE_VIDEO_HEVC;
1302 mCodecLooper = new ALooper;
1303 mCodecLooper->setName("Camera3-HeicComposite-MediaCodecLooper");
1304 status_t res = mCodecLooper->start(
1305 false, // runOnCallingThread
1306 false, // canCallJava
1307 PRIORITY_AUDIO);
1308 if (res != OK) {
1309 ALOGE("%s: Failed to start codec looper: %s (%d)",
1310 __FUNCTION__, strerror(-res), res);
1311 return NO_INIT;
1312 }
1313
1314 // Create HEIC/HEVC codec.
1315 if (mUseHeic) {
1316 mCodec = MediaCodec::CreateByType(mCodecLooper, desiredMime, true /*encoder*/);
1317 } else {
1318 mCodec = MediaCodec::CreateByComponentName(mCodecLooper, hevcName);
1319 }
1320 if (mCodec == nullptr) {
1321 ALOGE("%s: Failed to create codec for %s", __FUNCTION__, desiredMime);
1322 return NO_INIT;
1323 }
1324
1325 // Create Looper and handler for Codec callback.
1326 mCodecCallbackHandler = new CodecCallbackHandler(this);
1327 if (mCodecCallbackHandler == nullptr) {
1328 ALOGE("%s: Failed to create codec callback handler", __FUNCTION__);
1329 return NO_MEMORY;
1330 }
1331 mCallbackLooper = new ALooper;
1332 mCallbackLooper->setName("Camera3-HeicComposite-MediaCodecCallbackLooper");
1333 res = mCallbackLooper->start(
1334 false, // runOnCallingThread
1335 false, // canCallJava
1336 PRIORITY_AUDIO);
1337 if (res != OK) {
1338 ALOGE("%s: Failed to start media callback looper: %s (%d)",
1339 __FUNCTION__, strerror(-res), res);
1340 return NO_INIT;
1341 }
1342 mCallbackLooper->registerHandler(mCodecCallbackHandler);
1343
1344 mAsyncNotify = new AMessage(kWhatCallbackNotify, mCodecCallbackHandler);
1345 res = mCodec->setCallback(mAsyncNotify);
1346 if (res != OK) {
1347 ALOGE("%s: Failed to set MediaCodec callback: %s (%d)", __FUNCTION__,
1348 strerror(-res), res);
1349 return res;
1350 }
1351
1352 // Create output format and configure the Codec.
1353 sp<AMessage> outputFormat = new AMessage();
1354 outputFormat->setString(KEY_MIME, desiredMime);
1355 outputFormat->setInt32(KEY_BITRATE_MODE, BITRATE_MODE_CQ);
1356 outputFormat->setInt32(KEY_QUALITY, kDefaultJpegQuality);
1357 // Ask codec to skip timestamp check and encode all frames.
1358 outputFormat->setInt64(KEY_MAX_PTS_GAP_TO_ENCODER, kNoFrameDropMaxPtsGap);
1359
1360 int32_t gridWidth, gridHeight, gridRows, gridCols;
1361 if (useGrid || mUseHeic) {
1362 gridWidth = HeicEncoderInfoManager::kGridWidth;
1363 gridHeight = HeicEncoderInfoManager::kGridHeight;
1364 gridRows = (height + gridHeight - 1)/gridHeight;
1365 gridCols = (width + gridWidth - 1)/gridWidth;
1366
1367 if (mUseHeic) {
1368 outputFormat->setInt32(KEY_TILE_WIDTH, gridWidth);
1369 outputFormat->setInt32(KEY_TILE_HEIGHT, gridHeight);
1370 outputFormat->setInt32(KEY_GRID_COLUMNS, gridCols);
1371 outputFormat->setInt32(KEY_GRID_ROWS, gridRows);
1372 }
1373
1374 } else {
1375 gridWidth = width;
1376 gridHeight = height;
1377 gridRows = 1;
1378 gridCols = 1;
1379 }
1380
1381 outputFormat->setInt32(KEY_WIDTH, !useGrid ? width : gridWidth);
1382 outputFormat->setInt32(KEY_HEIGHT, !useGrid ? height : gridHeight);
1383 outputFormat->setInt32(KEY_I_FRAME_INTERVAL, 0);
1384 outputFormat->setInt32(KEY_COLOR_FORMAT,
1385 useGrid ? COLOR_FormatYUV420Flexible : COLOR_FormatSurface);
1386 outputFormat->setInt32(KEY_FRAME_RATE, useGrid ? gridRows * gridCols : kNoGridOpRate);
1387 // This only serves as a hint to encoder when encoding is not real-time.
1388 outputFormat->setInt32(KEY_OPERATING_RATE, useGrid ? kGridOpRate : kNoGridOpRate);
1389
1390 res = mCodec->configure(outputFormat, nullptr /*nativeWindow*/,
1391 nullptr /*crypto*/, CONFIGURE_FLAG_ENCODE);
1392 if (res != OK) {
1393 ALOGE("%s: Failed to configure codec: %s (%d)", __FUNCTION__,
1394 strerror(-res), res);
1395 return res;
1396 }
1397
1398 mGridWidth = gridWidth;
1399 mGridHeight = gridHeight;
1400 mGridRows = gridRows;
1401 mGridCols = gridCols;
1402 mUseGrid = useGrid;
1403 mOutputWidth = width;
1404 mOutputHeight = height;
1405 mAppSegmentMaxSize = calcAppSegmentMaxSize(cameraDevice->info());
1406 mMaxHeicBufferSize =
1407 ALIGN(mOutputWidth, HeicEncoderInfoManager::kGridWidth) *
1408 ALIGN(mOutputHeight, HeicEncoderInfoManager::kGridHeight) * 3 / 2 + mAppSegmentMaxSize;
1409
1410 return OK;
1411 }
1412
deinitCodec()1413 void HeicCompositeStream::deinitCodec() {
1414 ALOGV("%s", __FUNCTION__);
1415 if (mCodec != nullptr) {
1416 mCodec->stop();
1417 mCodec->release();
1418 mCodec.clear();
1419 }
1420
1421 if (mCodecLooper != nullptr) {
1422 mCodecLooper->stop();
1423 mCodecLooper.clear();
1424 }
1425
1426 if (mCallbackLooper != nullptr) {
1427 mCallbackLooper->stop();
1428 mCallbackLooper.clear();
1429 }
1430
1431 mAsyncNotify.clear();
1432 mFormat.clear();
1433 }
1434
1435 // Return the size of the complete list of app segment, 0 indicates failure
findAppSegmentsSize(const uint8_t * appSegmentBuffer,size_t maxSize,size_t * app1SegmentSize)1436 size_t HeicCompositeStream::findAppSegmentsSize(const uint8_t* appSegmentBuffer,
1437 size_t maxSize, size_t *app1SegmentSize) {
1438 if (appSegmentBuffer == nullptr || app1SegmentSize == nullptr) {
1439 ALOGE("%s: Invalid input appSegmentBuffer %p, app1SegmentSize %p",
1440 __FUNCTION__, appSegmentBuffer, app1SegmentSize);
1441 return 0;
1442 }
1443
1444 size_t expectedSize = 0;
1445 // First check for EXIF transport header at the end of the buffer
1446 const uint8_t *header = appSegmentBuffer + (maxSize - sizeof(CameraBlob));
1447 const CameraBlob *blob = (const CameraBlob*)(header);
1448 if (blob->blobId != CameraBlobId::JPEG_APP_SEGMENTS) {
1449 ALOGE("%s: Invalid EXIF blobId %d", __FUNCTION__, blob->blobId);
1450 return 0;
1451 }
1452
1453 expectedSize = blob->blobSizeBytes;
1454 if (expectedSize == 0 || expectedSize > maxSize - sizeof(CameraBlob)) {
1455 ALOGE("%s: Invalid blobSize %zu.", __FUNCTION__, expectedSize);
1456 return 0;
1457 }
1458
1459 uint32_t totalSize = 0;
1460
1461 // Verify APP1 marker (mandatory)
1462 uint8_t app1Marker[] = {0xFF, 0xE1};
1463 if (memcmp(appSegmentBuffer, app1Marker, sizeof(app1Marker))) {
1464 ALOGE("%s: Invalid APP1 marker: %x, %x", __FUNCTION__,
1465 appSegmentBuffer[0], appSegmentBuffer[1]);
1466 return 0;
1467 }
1468 totalSize += sizeof(app1Marker);
1469
1470 uint16_t app1Size = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
1471 appSegmentBuffer[totalSize+1];
1472 totalSize += app1Size;
1473
1474 ALOGV("%s: Expected APP segments size %zu, APP1 segment size %u",
1475 __FUNCTION__, expectedSize, app1Size);
1476 while (totalSize < expectedSize) {
1477 if (appSegmentBuffer[totalSize] != 0xFF ||
1478 appSegmentBuffer[totalSize+1] <= 0xE1 ||
1479 appSegmentBuffer[totalSize+1] > 0xEF) {
1480 // Invalid APPn marker
1481 ALOGE("%s: Invalid APPn marker: %x, %x", __FUNCTION__,
1482 appSegmentBuffer[totalSize], appSegmentBuffer[totalSize+1]);
1483 return 0;
1484 }
1485 totalSize += 2;
1486
1487 uint16_t appnSize = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
1488 appSegmentBuffer[totalSize+1];
1489 totalSize += appnSize;
1490 }
1491
1492 if (totalSize != expectedSize) {
1493 ALOGE("%s: Invalid JPEG APP segments: totalSize %u vs expected size %zu",
1494 __FUNCTION__, totalSize, expectedSize);
1495 return 0;
1496 }
1497
1498 *app1SegmentSize = app1Size + sizeof(app1Marker);
1499 return expectedSize;
1500 }
1501
copyOneYuvTile(sp<MediaCodecBuffer> & codecBuffer,const CpuConsumer::LockedBuffer & yuvBuffer,size_t top,size_t left,size_t width,size_t height)1502 status_t HeicCompositeStream::copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer,
1503 const CpuConsumer::LockedBuffer& yuvBuffer,
1504 size_t top, size_t left, size_t width, size_t height) {
1505 ATRACE_CALL();
1506
1507 // Get stride information for codecBuffer
1508 sp<ABuffer> imageData;
1509 if (!codecBuffer->meta()->findBuffer("image-data", &imageData)) {
1510 ALOGE("%s: Codec input buffer is not for image data!", __FUNCTION__);
1511 return BAD_VALUE;
1512 }
1513 if (imageData->size() != sizeof(MediaImage2)) {
1514 ALOGE("%s: Invalid codec input image size %zu, expected %zu",
1515 __FUNCTION__, imageData->size(), sizeof(MediaImage2));
1516 return BAD_VALUE;
1517 }
1518 MediaImage2* imageInfo = reinterpret_cast<MediaImage2*>(imageData->data());
1519 if (imageInfo->mType != MediaImage2::MEDIA_IMAGE_TYPE_YUV ||
1520 imageInfo->mBitDepth != 8 ||
1521 imageInfo->mBitDepthAllocated != 8 ||
1522 imageInfo->mNumPlanes != 3) {
1523 ALOGE("%s: Invalid codec input image info: mType %d, mBitDepth %d, "
1524 "mBitDepthAllocated %d, mNumPlanes %d!", __FUNCTION__,
1525 imageInfo->mType, imageInfo->mBitDepth,
1526 imageInfo->mBitDepthAllocated, imageInfo->mNumPlanes);
1527 return BAD_VALUE;
1528 }
1529
1530 ALOGV("%s: yuvBuffer chromaStep %d, chromaStride %d",
1531 __FUNCTION__, yuvBuffer.chromaStep, yuvBuffer.chromaStride);
1532 ALOGV("%s: U offset %u, V offset %u, U rowInc %d, V rowInc %d, U colInc %d, V colInc %d",
1533 __FUNCTION__, imageInfo->mPlane[MediaImage2::U].mOffset,
1534 imageInfo->mPlane[MediaImage2::V].mOffset,
1535 imageInfo->mPlane[MediaImage2::U].mRowInc,
1536 imageInfo->mPlane[MediaImage2::V].mRowInc,
1537 imageInfo->mPlane[MediaImage2::U].mColInc,
1538 imageInfo->mPlane[MediaImage2::V].mColInc);
1539
1540 // Y
1541 for (auto row = top; row < top+height; row++) {
1542 uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::Y].mOffset +
1543 imageInfo->mPlane[MediaImage2::Y].mRowInc * (row - top);
1544 mFnCopyRow(yuvBuffer.data+row*yuvBuffer.stride+left, dst, width);
1545 }
1546
1547 // U is Cb, V is Cr
1548 bool codecUPlaneFirst = imageInfo->mPlane[MediaImage2::V].mOffset >
1549 imageInfo->mPlane[MediaImage2::U].mOffset;
1550 uint32_t codecUvOffsetDiff = codecUPlaneFirst ?
1551 imageInfo->mPlane[MediaImage2::V].mOffset - imageInfo->mPlane[MediaImage2::U].mOffset :
1552 imageInfo->mPlane[MediaImage2::U].mOffset - imageInfo->mPlane[MediaImage2::V].mOffset;
1553 bool isCodecUvSemiplannar = (codecUvOffsetDiff == 1) &&
1554 (imageInfo->mPlane[MediaImage2::U].mRowInc ==
1555 imageInfo->mPlane[MediaImage2::V].mRowInc) &&
1556 (imageInfo->mPlane[MediaImage2::U].mColInc == 2) &&
1557 (imageInfo->mPlane[MediaImage2::V].mColInc == 2);
1558 bool isCodecUvPlannar =
1559 ((codecUPlaneFirst && codecUvOffsetDiff >=
1560 imageInfo->mPlane[MediaImage2::U].mRowInc * imageInfo->mHeight/2) ||
1561 ((!codecUPlaneFirst && codecUvOffsetDiff >=
1562 imageInfo->mPlane[MediaImage2::V].mRowInc * imageInfo->mHeight/2))) &&
1563 imageInfo->mPlane[MediaImage2::U].mColInc == 1 &&
1564 imageInfo->mPlane[MediaImage2::V].mColInc == 1;
1565 bool cameraUPlaneFirst = yuvBuffer.dataCr > yuvBuffer.dataCb;
1566
1567 if (isCodecUvSemiplannar && yuvBuffer.chromaStep == 2 &&
1568 (codecUPlaneFirst == cameraUPlaneFirst)) {
1569 // UV semiplannar
1570 // The chrome plane could be either Cb first, or Cr first. Take the
1571 // smaller address.
1572 uint8_t *src = std::min(yuvBuffer.dataCb, yuvBuffer.dataCr);
1573 MediaImage2::PlaneIndex dstPlane = codecUvOffsetDiff > 0 ? MediaImage2::U : MediaImage2::V;
1574 for (auto row = top/2; row < (top+height)/2; row++) {
1575 uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[dstPlane].mOffset +
1576 imageInfo->mPlane[dstPlane].mRowInc * (row - top/2);
1577 mFnCopyRow(src+row*yuvBuffer.chromaStride+left, dst, width);
1578 }
1579 } else if (isCodecUvPlannar && yuvBuffer.chromaStep == 1) {
1580 // U plane
1581 for (auto row = top/2; row < (top+height)/2; row++) {
1582 uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::U].mOffset +
1583 imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2);
1584 mFnCopyRow(yuvBuffer.dataCb+row*yuvBuffer.chromaStride+left/2, dst, width/2);
1585 }
1586
1587 // V plane
1588 for (auto row = top/2; row < (top+height)/2; row++) {
1589 uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::V].mOffset +
1590 imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2);
1591 mFnCopyRow(yuvBuffer.dataCr+row*yuvBuffer.chromaStride+left/2, dst, width/2);
1592 }
1593 } else {
1594 // Convert between semiplannar and plannar, or when UV orders are
1595 // different.
1596 uint8_t *dst = codecBuffer->data();
1597 for (auto row = top/2; row < (top+height)/2; row++) {
1598 for (auto col = left/2; col < (left+width)/2; col++) {
1599 // U/Cb
1600 int32_t dstIndex = imageInfo->mPlane[MediaImage2::U].mOffset +
1601 imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2) +
1602 imageInfo->mPlane[MediaImage2::U].mColInc * (col - left/2);
1603 int32_t srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
1604 dst[dstIndex] = yuvBuffer.dataCb[srcIndex];
1605
1606 // V/Cr
1607 dstIndex = imageInfo->mPlane[MediaImage2::V].mOffset +
1608 imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2) +
1609 imageInfo->mPlane[MediaImage2::V].mColInc * (col - left/2);
1610 srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
1611 dst[dstIndex] = yuvBuffer.dataCr[srcIndex];
1612 }
1613 }
1614 }
1615 return OK;
1616 }
1617
initCopyRowFunction(int32_t width)1618 void HeicCompositeStream::initCopyRowFunction([[maybe_unused]] int32_t width)
1619 {
1620 using namespace libyuv;
1621
1622 mFnCopyRow = CopyRow_C;
1623 #if defined(HAS_COPYROW_SSE2)
1624 if (TestCpuFlag(kCpuHasSSE2)) {
1625 mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
1626 }
1627 #endif
1628 #if defined(HAS_COPYROW_AVX)
1629 if (TestCpuFlag(kCpuHasAVX)) {
1630 mFnCopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
1631 }
1632 #endif
1633 #if defined(HAS_COPYROW_ERMS)
1634 if (TestCpuFlag(kCpuHasERMS)) {
1635 mFnCopyRow = CopyRow_ERMS;
1636 }
1637 #endif
1638 #if defined(HAS_COPYROW_NEON)
1639 if (TestCpuFlag(kCpuHasNEON)) {
1640 mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
1641 }
1642 #endif
1643 #if defined(HAS_COPYROW_MIPS)
1644 if (TestCpuFlag(kCpuHasMIPS)) {
1645 mFnCopyRow = CopyRow_MIPS;
1646 }
1647 #endif
1648 }
1649
calcAppSegmentMaxSize(const CameraMetadata & info)1650 size_t HeicCompositeStream::calcAppSegmentMaxSize(const CameraMetadata& info) {
1651 camera_metadata_ro_entry_t entry = info.find(ANDROID_HEIC_INFO_MAX_JPEG_APP_SEGMENTS_COUNT);
1652 size_t maxAppsSegment = 1;
1653 if (entry.count > 0) {
1654 maxAppsSegment = entry.data.u8[0] < 1 ? 1 :
1655 entry.data.u8[0] > 16 ? 16 : entry.data.u8[0];
1656 }
1657 return maxAppsSegment * (2 + 0xFFFF) + sizeof(CameraBlob);
1658 }
1659
updateCodecQualityLocked(int32_t quality)1660 void HeicCompositeStream::updateCodecQualityLocked(int32_t quality) {
1661 if (quality != mQuality) {
1662 sp<AMessage> qualityParams = new AMessage;
1663 qualityParams->setInt32(PARAMETER_KEY_VIDEO_BITRATE, quality);
1664 status_t res = mCodec->setParameters(qualityParams);
1665 if (res != OK) {
1666 ALOGE("%s: Failed to set codec quality: %s (%d)",
1667 __FUNCTION__, strerror(-res), res);
1668 } else {
1669 mQuality = quality;
1670 }
1671 }
1672 }
1673
threadLoop()1674 bool HeicCompositeStream::threadLoop() {
1675 int64_t frameNumber = -1;
1676 bool newInputAvailable = false;
1677
1678 {
1679 Mutex::Autolock l(mMutex);
1680 if (mErrorState) {
1681 // In case we landed in error state, return any pending buffers and
1682 // halt all further processing.
1683 compilePendingInputLocked();
1684 releaseInputFramesLocked();
1685 return false;
1686 }
1687
1688
1689 while (!newInputAvailable) {
1690 compilePendingInputLocked();
1691 newInputAvailable = getNextReadyInputLocked(&frameNumber);
1692
1693 if (!newInputAvailable) {
1694 auto failingFrameNumber = getNextFailingInputLocked();
1695 if (failingFrameNumber >= 0) {
1696 releaseInputFrameLocked(failingFrameNumber,
1697 &mPendingInputFrames[failingFrameNumber]);
1698
1699 // It's okay to remove the entry from mPendingInputFrames
1700 // because:
1701 // 1. Only one internal stream (main input) is critical in
1702 // backing the output stream.
1703 // 2. If captureResult/appSegment arrives after the entry is
1704 // removed, they are simply skipped.
1705 mPendingInputFrames.erase(failingFrameNumber);
1706 if (mPendingInputFrames.size() == 0) {
1707 markTrackerIdle();
1708 }
1709 return true;
1710 }
1711
1712 auto ret = mInputReadyCondition.waitRelative(mMutex, kWaitDuration);
1713 if (ret == TIMED_OUT) {
1714 return true;
1715 } else if (ret != OK) {
1716 ALOGE("%s: Timed wait on condition failed: %s (%d)", __FUNCTION__,
1717 strerror(-ret), ret);
1718 return false;
1719 }
1720 }
1721 }
1722 }
1723
1724 auto res = processInputFrame(frameNumber, mPendingInputFrames[frameNumber]);
1725 Mutex::Autolock l(mMutex);
1726 if (res != OK) {
1727 ALOGE("%s: Failed processing frame with timestamp: %" PRIu64 ", frameNumber: %"
1728 PRId64 ": %s (%d)", __FUNCTION__, mPendingInputFrames[frameNumber].timestamp,
1729 frameNumber, strerror(-res), res);
1730 mPendingInputFrames[frameNumber].error = true;
1731 }
1732
1733 releaseInputFramesLocked();
1734
1735 return true;
1736 }
1737
flagAnExifErrorFrameNumber(int64_t frameNumber)1738 void HeicCompositeStream::flagAnExifErrorFrameNumber(int64_t frameNumber) {
1739 Mutex::Autolock l(mMutex);
1740 mExifErrorFrameNumbers.emplace(frameNumber);
1741 mInputReadyCondition.signal();
1742 }
1743
onStreamBufferError(const CaptureResultExtras & resultExtras)1744 bool HeicCompositeStream::onStreamBufferError(const CaptureResultExtras& resultExtras) {
1745 bool res = false;
1746 int64_t frameNumber = resultExtras.frameNumber;
1747
1748 // Buffer errors concerning internal composite streams should not be directly visible to
1749 // camera clients. They must only receive a single buffer error with the public composite
1750 // stream id.
1751 if (resultExtras.errorStreamId == mAppSegmentStreamId) {
1752 ALOGV("%s: APP_SEGMENT frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1753 flagAnExifErrorFrameNumber(frameNumber);
1754 res = true;
1755 } else if (resultExtras.errorStreamId == mMainImageStreamId) {
1756 ALOGV("%s: YUV frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1757 flagAnErrorFrameNumber(frameNumber);
1758 res = true;
1759 }
1760
1761 return res;
1762 }
1763
onResultError(const CaptureResultExtras & resultExtras)1764 void HeicCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {
1765 // For result error, since the APPS_SEGMENT buffer already contains EXIF,
1766 // simply skip using the capture result metadata to override EXIF.
1767 Mutex::Autolock l(mMutex);
1768
1769 int64_t timestamp = -1;
1770 for (const auto& fn : mSettingsByFrameNumber) {
1771 if (fn.first == resultExtras.frameNumber) {
1772 timestamp = fn.second.timestamp;
1773 break;
1774 }
1775 }
1776 if (timestamp == -1) {
1777 for (const auto& inputFrame : mPendingInputFrames) {
1778 if (inputFrame.first == resultExtras.frameNumber) {
1779 timestamp = inputFrame.second.timestamp;
1780 break;
1781 }
1782 }
1783 }
1784
1785 if (timestamp == -1) {
1786 ALOGE("%s: Failed to find shutter timestamp for result error!", __FUNCTION__);
1787 return;
1788 }
1789
1790 mCaptureResults.emplace(timestamp, std::make_tuple(resultExtras.frameNumber, CameraMetadata()));
1791 ALOGV("%s: timestamp %" PRId64 ", frameNumber %" PRId64, __FUNCTION__,
1792 timestamp, resultExtras.frameNumber);
1793 mInputReadyCondition.signal();
1794 }
1795
onRequestError(const CaptureResultExtras & resultExtras)1796 void HeicCompositeStream::onRequestError(const CaptureResultExtras& resultExtras) {
1797 auto frameNumber = resultExtras.frameNumber;
1798 ALOGV("%s: frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1799 Mutex::Autolock l(mMutex);
1800 auto numRequests = mSettingsByFrameNumber.erase(frameNumber);
1801 if (numRequests == 0) {
1802 // Pending request has been populated into mPendingInputFrames
1803 mErrorFrameNumbers.emplace(frameNumber);
1804 mInputReadyCondition.signal();
1805 } else {
1806 // REQUEST_ERROR was received without onShutter.
1807 }
1808 }
1809
markTrackerIdle()1810 void HeicCompositeStream::markTrackerIdle() {
1811 sp<StatusTracker> statusTracker = mStatusTracker.promote();
1812 if (statusTracker != nullptr) {
1813 statusTracker->markComponentIdle(mStatusId, Fence::NO_FENCE);
1814 ALOGV("%s: Mark component as idle", __FUNCTION__);
1815 }
1816 }
1817
onMessageReceived(const sp<AMessage> & msg)1818 void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMessage> &msg) {
1819 sp<HeicCompositeStream> parent = mParent.promote();
1820 if (parent == nullptr) return;
1821
1822 switch (msg->what()) {
1823 case kWhatCallbackNotify: {
1824 int32_t cbID;
1825 if (!msg->findInt32("callbackID", &cbID)) {
1826 ALOGE("kWhatCallbackNotify: callbackID is expected.");
1827 break;
1828 }
1829
1830 ALOGV("kWhatCallbackNotify: cbID = %d", cbID);
1831
1832 switch (cbID) {
1833 case MediaCodec::CB_INPUT_AVAILABLE: {
1834 int32_t index;
1835 if (!msg->findInt32("index", &index)) {
1836 ALOGE("CB_INPUT_AVAILABLE: index is expected.");
1837 break;
1838 }
1839 parent->onHeicInputFrameAvailable(index);
1840 break;
1841 }
1842
1843 case MediaCodec::CB_OUTPUT_AVAILABLE: {
1844 int32_t index;
1845 size_t offset;
1846 size_t size;
1847 int64_t timeUs;
1848 int32_t flags;
1849
1850 if (!msg->findInt32("index", &index)) {
1851 ALOGE("CB_OUTPUT_AVAILABLE: index is expected.");
1852 break;
1853 }
1854 if (!msg->findSize("offset", &offset)) {
1855 ALOGE("CB_OUTPUT_AVAILABLE: offset is expected.");
1856 break;
1857 }
1858 if (!msg->findSize("size", &size)) {
1859 ALOGE("CB_OUTPUT_AVAILABLE: size is expected.");
1860 break;
1861 }
1862 if (!msg->findInt64("timeUs", &timeUs)) {
1863 ALOGE("CB_OUTPUT_AVAILABLE: timeUs is expected.");
1864 break;
1865 }
1866 if (!msg->findInt32("flags", &flags)) {
1867 ALOGE("CB_OUTPUT_AVAILABLE: flags is expected.");
1868 break;
1869 }
1870
1871 CodecOutputBufferInfo bufferInfo = {
1872 index,
1873 (int32_t)offset,
1874 (int32_t)size,
1875 timeUs,
1876 (uint32_t)flags};
1877
1878 parent->onHeicOutputFrameAvailable(bufferInfo);
1879 break;
1880 }
1881
1882 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: {
1883 sp<AMessage> format;
1884 if (!msg->findMessage("format", &format)) {
1885 ALOGE("CB_OUTPUT_FORMAT_CHANGED: format is expected.");
1886 break;
1887 }
1888 // Here format is MediaCodec's internal copy of output format.
1889 // Make a copy since onHeicFormatChanged() might modify it.
1890 sp<AMessage> formatCopy;
1891 if (format != nullptr) {
1892 formatCopy = format->dup();
1893 }
1894 parent->onHeicFormatChanged(formatCopy);
1895 break;
1896 }
1897
1898 case MediaCodec::CB_ERROR: {
1899 status_t err;
1900 int32_t actionCode;
1901 AString detail;
1902 if (!msg->findInt32("err", &err)) {
1903 ALOGE("CB_ERROR: err is expected.");
1904 break;
1905 }
1906 if (!msg->findInt32("action", &actionCode)) {
1907 ALOGE("CB_ERROR: action is expected.");
1908 break;
1909 }
1910 msg->findString("detail", &detail);
1911 ALOGE("Codec reported error(0x%x), actionCode(%d), detail(%s)",
1912 err, actionCode, detail.c_str());
1913
1914 parent->onHeicCodecError();
1915 break;
1916 }
1917
1918 default: {
1919 ALOGE("kWhatCallbackNotify: callbackID(%d) is unexpected.", cbID);
1920 break;
1921 }
1922 }
1923 break;
1924 }
1925
1926 default:
1927 ALOGE("shouldn't be here");
1928 break;
1929 }
1930 }
1931
1932 }; // namespace camera3
1933 }; // namespace android
1934