1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "ExtCamUtils"
18 // #define LOG_NDEBUG 0
19
20 #include "ExternalCameraUtils.h"
21
22 #include <aidlcommonsupport/NativeHandle.h>
23 #include <jpeglib.h>
24 #include <linux/videodev2.h>
25 #include <log/log.h>
26 #include <algorithm>
27 #include <cinttypes>
28 #include <cmath>
29
30 #define HAVE_JPEG // required for libyuv.h to export MJPEG decode APIs
31 #include <libyuv.h>
32
33 namespace android {
34 namespace hardware {
35 namespace camera {
36
37 namespace external {
38 namespace common {
39
40 namespace {
41 const int kDefaultCameraIdOffset = 100;
42 const int kDefaultJpegBufSize = 5 << 20; // 5MB
43 const int kDefaultNumVideoBuffer = 4;
44 const int kDefaultNumStillBuffer = 2;
45 const int kDefaultOrientation = 0; // suitable for natural landscape displays like tablet/TV
46 // For phone devices 270 is better
47 } // anonymous namespace
48
49 const char* ExternalCameraConfig::kDefaultCfgPath = "/vendor/etc/external_camera_config.xml";
50
loadFromCfg(const char * cfgPath)51 ExternalCameraConfig ExternalCameraConfig::loadFromCfg(const char* cfgPath) {
52 using namespace tinyxml2;
53 ExternalCameraConfig ret;
54
55 XMLDocument configXml;
56 XMLError err = configXml.LoadFile(cfgPath);
57 if (err != XML_SUCCESS) {
58 ALOGE("%s: Unable to load external camera config file '%s'. Error: %s", __FUNCTION__,
59 cfgPath, XMLDocument::ErrorIDToName(err));
60 return ret;
61 } else {
62 ALOGI("%s: load external camera config succeeded!", __FUNCTION__);
63 }
64
65 XMLElement* extCam = configXml.FirstChildElement("ExternalCamera");
66 if (extCam == nullptr) {
67 ALOGI("%s: no external camera config specified", __FUNCTION__);
68 return ret;
69 }
70
71 XMLElement* providerCfg = extCam->FirstChildElement("Provider");
72 if (providerCfg == nullptr) {
73 ALOGI("%s: no external camera provider config specified", __FUNCTION__);
74 return ret;
75 }
76
77 XMLElement* cameraIdOffset = providerCfg->FirstChildElement("CameraIdOffset");
78 if (cameraIdOffset != nullptr) {
79 ret.cameraIdOffset = std::atoi(cameraIdOffset->GetText());
80 }
81
82 XMLElement* ignore = providerCfg->FirstChildElement("ignore");
83 if (ignore == nullptr) {
84 ALOGI("%s: no internal ignored device specified", __FUNCTION__);
85 return ret;
86 }
87
88 XMLElement* id = ignore->FirstChildElement("id");
89 while (id != nullptr) {
90 const char* text = id->GetText();
91 if (text != nullptr) {
92 ret.mInternalDevices.insert(text);
93 ALOGI("%s: device %s will be ignored by external camera provider", __FUNCTION__, text);
94 }
95 id = id->NextSiblingElement("id");
96 }
97
98 XMLElement* deviceCfg = extCam->FirstChildElement("Device");
99 if (deviceCfg == nullptr) {
100 ALOGI("%s: no external camera device config specified", __FUNCTION__);
101 return ret;
102 }
103
104 XMLElement* jpegBufSz = deviceCfg->FirstChildElement("MaxJpegBufferSize");
105 if (jpegBufSz == nullptr) {
106 ALOGI("%s: no max jpeg buffer size specified", __FUNCTION__);
107 } else {
108 ret.maxJpegBufSize = jpegBufSz->UnsignedAttribute("bytes", /*Default*/ kDefaultJpegBufSize);
109 }
110
111 XMLElement* numVideoBuf = deviceCfg->FirstChildElement("NumVideoBuffers");
112 if (numVideoBuf == nullptr) {
113 ALOGI("%s: no num video buffers specified", __FUNCTION__);
114 } else {
115 ret.numVideoBuffers =
116 numVideoBuf->UnsignedAttribute("count", /*Default*/ kDefaultNumVideoBuffer);
117 }
118
119 XMLElement* numStillBuf = deviceCfg->FirstChildElement("NumStillBuffers");
120 if (numStillBuf == nullptr) {
121 ALOGI("%s: no num still buffers specified", __FUNCTION__);
122 } else {
123 ret.numStillBuffers =
124 numStillBuf->UnsignedAttribute("count", /*Default*/ kDefaultNumStillBuffer);
125 }
126
127 XMLElement* fpsList = deviceCfg->FirstChildElement("FpsList");
128 if (fpsList == nullptr) {
129 ALOGI("%s: no fps list specified", __FUNCTION__);
130 } else {
131 if (!updateFpsList(fpsList, ret.fpsLimits)) {
132 return ret;
133 }
134 }
135
136 XMLElement* depth = deviceCfg->FirstChildElement("Depth16Supported");
137 if (depth == nullptr) {
138 ret.depthEnabled = false;
139 ALOGI("%s: depth output is not enabled", __FUNCTION__);
140 } else {
141 ret.depthEnabled = depth->BoolAttribute("enabled", false);
142 }
143
144 if (ret.depthEnabled) {
145 XMLElement* depthFpsList = deviceCfg->FirstChildElement("DepthFpsList");
146 if (depthFpsList == nullptr) {
147 ALOGW("%s: no depth fps list specified", __FUNCTION__);
148 } else {
149 if (!updateFpsList(depthFpsList, ret.depthFpsLimits)) {
150 return ret;
151 }
152 }
153 }
154
155 XMLElement* minStreamSize = deviceCfg->FirstChildElement("MinimumStreamSize");
156 if (minStreamSize == nullptr) {
157 ALOGI("%s: no minimum stream size specified", __FUNCTION__);
158 } else {
159 ret.minStreamSize = {
160 static_cast<int32_t>(minStreamSize->UnsignedAttribute("width", /*Default*/ 0)),
161 static_cast<int32_t>(minStreamSize->UnsignedAttribute("height", /*Default*/ 0))};
162 }
163
164 XMLElement* orientation = deviceCfg->FirstChildElement("Orientation");
165 if (orientation == nullptr) {
166 ALOGI("%s: no sensor orientation specified", __FUNCTION__);
167 } else {
168 ret.orientation = orientation->IntAttribute("degree", /*Default*/ kDefaultOrientation);
169 }
170
171 ALOGI("%s: external camera cfg loaded: maxJpgBufSize %d,"
172 " num video buffers %d, num still buffers %d, orientation %d",
173 __FUNCTION__, ret.maxJpegBufSize, ret.numVideoBuffers, ret.numStillBuffers,
174 ret.orientation);
175 for (const auto& limit : ret.fpsLimits) {
176 ALOGI("%s: fpsLimitList: %dx%d@%f", __FUNCTION__, limit.size.width, limit.size.height,
177 limit.fpsUpperBound);
178 }
179 for (const auto& limit : ret.depthFpsLimits) {
180 ALOGI("%s: depthFpsLimitList: %dx%d@%f", __FUNCTION__, limit.size.width, limit.size.height,
181 limit.fpsUpperBound);
182 }
183 ALOGI("%s: minStreamSize: %dx%d", __FUNCTION__, ret.minStreamSize.width,
184 ret.minStreamSize.height);
185 return ret;
186 }
187
updateFpsList(tinyxml2::XMLElement * fpsList,std::vector<FpsLimitation> & fpsLimits)188 bool ExternalCameraConfig::updateFpsList(tinyxml2::XMLElement* fpsList,
189 std::vector<FpsLimitation>& fpsLimits) {
190 using namespace tinyxml2;
191 std::vector<FpsLimitation> limits;
192 XMLElement* row = fpsList->FirstChildElement("Limit");
193 while (row != nullptr) {
194 FpsLimitation prevLimit{{0, 0}, 1000.0};
195 FpsLimitation limit = {
196 {/* width */ static_cast<int32_t>(row->UnsignedAttribute("width", /*Default*/ 0)),
197 /* height */ static_cast<int32_t>(
198 row->UnsignedAttribute("height", /*Default*/ 0))},
199 /* fpsUpperBound */ row->DoubleAttribute("fpsBound", /*Default*/ 1000.0)};
200 if (limit.size.width <= prevLimit.size.width ||
201 limit.size.height <= prevLimit.size.height ||
202 limit.fpsUpperBound >= prevLimit.fpsUpperBound) {
203 ALOGE("%s: FPS limit list must have increasing size and decreasing fps!"
204 " Prev %dx%d@%f, Current %dx%d@%f",
205 __FUNCTION__, prevLimit.size.width, prevLimit.size.height,
206 prevLimit.fpsUpperBound, limit.size.width, limit.size.height,
207 limit.fpsUpperBound);
208 return false;
209 }
210 limits.push_back(limit);
211 row = row->NextSiblingElement("Limit");
212 }
213 fpsLimits = limits;
214 return true;
215 }
216
ExternalCameraConfig()217 ExternalCameraConfig::ExternalCameraConfig()
218 : cameraIdOffset(kDefaultCameraIdOffset),
219 maxJpegBufSize(kDefaultJpegBufSize),
220 numVideoBuffers(kDefaultNumVideoBuffer),
221 numStillBuffers(kDefaultNumStillBuffer),
222 depthEnabled(false),
223 orientation(kDefaultOrientation) {
224 fpsLimits.push_back({/* size */ {/* width */ 640, /* height */ 480}, /* fpsUpperBound */ 30.0});
225 fpsLimits.push_back({/* size */ {/* width */ 1280, /* height */ 720}, /* fpsUpperBound */ 7.5});
226 fpsLimits.push_back(
227 {/* size */ {/* width */ 1920, /* height */ 1080}, /* fpsUpperBound */ 5.0});
228 minStreamSize = {0, 0};
229 }
230
231 } // namespace common
232 } // namespace external
233
234 namespace device {
235 namespace implementation {
236
getFramesPerSecond() const237 double SupportedV4L2Format::FrameRate::getFramesPerSecond() const {
238 return static_cast<double>(durationDenominator) / durationNumerator;
239 }
240
Frame(uint32_t width,uint32_t height,uint32_t fourcc)241 Frame::Frame(uint32_t width, uint32_t height, uint32_t fourcc)
242 : mWidth(width), mHeight(height), mFourcc(fourcc) {}
~Frame()243 Frame::~Frame() {}
244
V4L2Frame(uint32_t w,uint32_t h,uint32_t fourcc,int bufIdx,int fd,uint32_t dataSize,uint64_t offset)245 V4L2Frame::V4L2Frame(uint32_t w, uint32_t h, uint32_t fourcc, int bufIdx, int fd, uint32_t dataSize,
246 uint64_t offset)
247 : Frame(w, h, fourcc), mBufferIndex(bufIdx), mFd(fd), mDataSize(dataSize), mOffset(offset) {}
248
~V4L2Frame()249 V4L2Frame::~V4L2Frame() {
250 unmap();
251 }
252
getData(uint8_t ** outData,size_t * dataSize)253 int V4L2Frame::getData(uint8_t** outData, size_t* dataSize) {
254 return map(outData, dataSize);
255 }
256
map(uint8_t ** data,size_t * dataSize)257 int V4L2Frame::map(uint8_t** data, size_t* dataSize) {
258 if (data == nullptr || dataSize == nullptr) {
259 ALOGI("%s: V4L2 buffer map bad argument: data %p, dataSize %p", __FUNCTION__, data,
260 dataSize);
261 return -EINVAL;
262 }
263
264 std::lock_guard<std::mutex> lk(mLock);
265 if (!mMapped) {
266 void* addr = mmap(nullptr, mDataSize, PROT_READ, MAP_SHARED, mFd, mOffset);
267 if (addr == MAP_FAILED) {
268 ALOGE("%s: V4L2 buffer map failed: %s", __FUNCTION__, strerror(errno));
269 return -EINVAL;
270 }
271 mData = static_cast<uint8_t*>(addr);
272 mMapped = true;
273 }
274 *data = mData;
275 *dataSize = mDataSize;
276 ALOGV("%s: V4L map FD %d, data %p size %zu", __FUNCTION__, mFd, mData, mDataSize);
277 return 0;
278 }
279
unmap()280 int V4L2Frame::unmap() {
281 std::lock_guard<std::mutex> lk(mLock);
282 if (mMapped) {
283 ALOGV("%s: V4L unmap data %p size %zu", __FUNCTION__, mData, mDataSize);
284 if (munmap(mData, mDataSize) != 0) {
285 ALOGE("%s: V4L2 buffer unmap failed: %s", __FUNCTION__, strerror(errno));
286 return -EINVAL;
287 }
288 mMapped = false;
289 }
290 return 0;
291 }
292
AllocatedFrame(uint32_t w,uint32_t h)293 AllocatedFrame::AllocatedFrame(uint32_t w, uint32_t h) : Frame(w, h, V4L2_PIX_FMT_YUV420) {}
~AllocatedFrame()294 AllocatedFrame::~AllocatedFrame() {}
295
getData(uint8_t ** outData,size_t * dataSize)296 int AllocatedFrame::getData(uint8_t** outData, size_t* dataSize) {
297 YCbCrLayout layout;
298 int ret = allocate(&layout);
299 if (ret != 0) {
300 return ret;
301 }
302 *outData = mData.data();
303 *dataSize = mBufferSize;
304 return 0;
305 }
306
allocate(YCbCrLayout * out)307 int AllocatedFrame::allocate(YCbCrLayout* out) {
308 std::lock_guard<std::mutex> lk(mLock);
309 if ((mWidth % 2) || (mHeight % 2)) {
310 ALOGE("%s: bad dimension %dx%d (not multiple of 2)", __FUNCTION__, mWidth, mHeight);
311 return -EINVAL;
312 }
313
314 // This frame might be sent to jpeglib to be encoded. Since AllocatedFrame only contains YUV420,
315 // jpeglib expects height and width of Y component to be an integral multiple of 2*DCTSIZE,
316 // and heights and widths of Cb and Cr components to be an integral multiple of DCTSIZE. If the
317 // image size does not meet this requirement, libjpeg expects its input to be padded to meet the
318 // constraints. This padding is removed from the final encoded image so the content in the
319 // padding doesn't matter. What matters is that the memory is accessible to jpeglib at the time
320 // of encoding.
321 // For example, if the image size is 1500x844 and DCTSIZE is 8, jpeglib expects a YUV 420
322 // frame with components of following sizes:
323 // Y: 1504x848 because 1504 and 848 are the next smallest multiples of 2*8
324 // Cb/Cr: 752x424 which are the next smallest multiples of 8
325
326 // jpeglib takes an array of row pointers which makes vertical padding trivial when setting up
327 // the pointers. Padding horizontally is a bit more complicated. AllocatedFrame holds the data
328 // in a flattened buffer, which means memory accesses past a row will flow into the next logical
329 // row. For any row of a component, we can consider the first few bytes of the next row as
330 // padding for the current one. This is true for Y and Cb components and all but last row of the
331 // Cr component. Reading past the last row of Cr component will lead to undefined behavior as
332 // libjpeg attempts to read memory past the allocated buffer. To prevent undefined behavior,
333 // the buffer allocated here is padded such that libjpeg never accesses unallocated memory when
334 // reading the last row. Effectively, we only need to ensure that the last row of Cr component
335 // has width that is an integral multiple of DCTSIZE.
336
337 size_t dataSize = mWidth * mHeight * 3 / 2; // YUV420
338
339 size_t cbWidth = mWidth / 2;
340 size_t requiredCbWidth = DCTSIZE * ((cbWidth + DCTSIZE - 1) / DCTSIZE);
341 size_t padding = requiredCbWidth - cbWidth;
342 size_t finalSize = dataSize + padding;
343
344 if (mData.size() != finalSize) {
345 mData.resize(finalSize);
346 mBufferSize = dataSize;
347 }
348
349 if (out != nullptr) {
350 out->y = mData.data();
351 out->yStride = mWidth;
352 uint8_t* cbStart = mData.data() + mWidth * mHeight;
353 uint8_t* crStart = cbStart + mWidth * mHeight / 4;
354 out->cb = cbStart;
355 out->cr = crStart;
356 out->cStride = mWidth / 2;
357 out->chromaStep = 1;
358 }
359 return 0;
360 }
361
getLayout(YCbCrLayout * out)362 int AllocatedFrame::getLayout(YCbCrLayout* out) {
363 IMapper::Rect noCrop = {0, 0, static_cast<int32_t>(mWidth), static_cast<int32_t>(mHeight)};
364 return getCroppedLayout(noCrop, out);
365 }
366
getCroppedLayout(const IMapper::Rect & rect,YCbCrLayout * out)367 int AllocatedFrame::getCroppedLayout(const IMapper::Rect& rect, YCbCrLayout* out) {
368 if (out == nullptr) {
369 ALOGE("%s: null out", __FUNCTION__);
370 return -1;
371 }
372
373 std::lock_guard<std::mutex> lk(mLock);
374 if ((rect.left + rect.width) > static_cast<int>(mWidth) ||
375 (rect.top + rect.height) > static_cast<int>(mHeight) || (rect.left % 2) || (rect.top % 2) ||
376 (rect.width % 2) || (rect.height % 2)) {
377 ALOGE("%s: bad rect left %d top %d w %d h %d", __FUNCTION__, rect.left, rect.top,
378 rect.width, rect.height);
379 return -1;
380 }
381
382 out->y = mData.data() + mWidth * rect.top + rect.left;
383 out->yStride = mWidth;
384 uint8_t* cbStart = mData.data() + mWidth * mHeight;
385 uint8_t* crStart = cbStart + mWidth * mHeight / 4;
386 out->cb = cbStart + mWidth * rect.top / 4 + rect.left / 2;
387 out->cr = crStart + mWidth * rect.top / 4 + rect.left / 2;
388 out->cStride = mWidth / 2;
389 out->chromaStep = 1;
390 return 0;
391 }
392
isAspectRatioClose(float ar1,float ar2)393 bool isAspectRatioClose(float ar1, float ar2) {
394 constexpr float kAspectRatioMatchThres = 0.025f; // This threshold is good enough to
395 // distinguish 4:3/16:9/20:9 1.33/1.78/2
396 return std::abs(ar1 - ar2) < kAspectRatioMatchThres;
397 }
398
importBufferImpl(std::map<int,CirculatingBuffers> & circulatingBuffers,HandleImporter & handleImporter,int32_t streamId,uint64_t bufId,buffer_handle_t buf,buffer_handle_t ** outBufPtr)399 aidl::android::hardware::camera::common::Status importBufferImpl(
400 /*inout*/ std::map<int, CirculatingBuffers>& circulatingBuffers,
401 /*inout*/ HandleImporter& handleImporter, int32_t streamId, uint64_t bufId,
402 buffer_handle_t buf,
403 /*out*/ buffer_handle_t** outBufPtr) {
404 using ::aidl::android::hardware::camera::common::Status;
405 if (buf == nullptr && bufId == BUFFER_ID_NO_BUFFER) {
406 ALOGE("%s: bufferId %" PRIu64 " has null buffer handle!", __FUNCTION__, bufId);
407 return Status::ILLEGAL_ARGUMENT;
408 }
409
410 CirculatingBuffers& cbs = circulatingBuffers[streamId];
411 if (cbs.count(bufId) == 0) {
412 if (buf == nullptr) {
413 ALOGE("%s: bufferId %" PRIu64 " has null buffer handle!", __FUNCTION__, bufId);
414 return Status::ILLEGAL_ARGUMENT;
415 }
416 // Register a newly seen buffer
417 buffer_handle_t importedBuf = buf;
418 handleImporter.importBuffer(importedBuf);
419 if (importedBuf == nullptr) {
420 ALOGE("%s: output buffer for stream %d is invalid!", __FUNCTION__, streamId);
421 return Status::INTERNAL_ERROR;
422 } else {
423 cbs[bufId] = importedBuf;
424 }
425 }
426 *outBufPtr = &cbs[bufId];
427 return Status::OK;
428 }
429
getFourCcFromLayout(const YCbCrLayout & layout)430 uint32_t getFourCcFromLayout(const YCbCrLayout& layout) {
431 intptr_t cb = reinterpret_cast<intptr_t>(layout.cb);
432 intptr_t cr = reinterpret_cast<intptr_t>(layout.cr);
433 if (std::abs(cb - cr) == 1 && layout.chromaStep == 2) {
434 // Interleaved format
435 if (layout.cb > layout.cr) {
436 return V4L2_PIX_FMT_NV21;
437 } else {
438 return V4L2_PIX_FMT_NV12;
439 }
440 } else if (layout.chromaStep == 1) {
441 // Planar format
442 if (layout.cb > layout.cr) {
443 return V4L2_PIX_FMT_YVU420; // YV12
444 } else {
445 return V4L2_PIX_FMT_YUV420; // YU12
446 }
447 } else {
448 return FLEX_YUV_GENERIC;
449 }
450 }
451
getCropRect(CroppingType ct,const Size & inSize,const Size & outSize,IMapper::Rect * out)452 int getCropRect(CroppingType ct, const Size& inSize, const Size& outSize, IMapper::Rect* out) {
453 if (out == nullptr) {
454 ALOGE("%s: out is null", __FUNCTION__);
455 return -1;
456 }
457
458 uint32_t inW = inSize.width;
459 uint32_t inH = inSize.height;
460 uint32_t outW = outSize.width;
461 uint32_t outH = outSize.height;
462
463 // Handle special case where aspect ratio is close to input but scaled
464 // dimension is slightly larger than input
465 float arIn = ASPECT_RATIO(inSize);
466 float arOut = ASPECT_RATIO(outSize);
467 if (isAspectRatioClose(arIn, arOut)) {
468 out->left = 0;
469 out->top = 0;
470 out->width = static_cast<int32_t>(inW);
471 out->height = static_cast<int32_t>(inH);
472 return 0;
473 }
474
475 if (ct == VERTICAL) {
476 uint64_t scaledOutH = static_cast<uint64_t>(outH) * inW / outW;
477 if (scaledOutH > inH) {
478 ALOGE("%s: Output size %dx%d cannot be vertically cropped from input size %dx%d",
479 __FUNCTION__, outW, outH, inW, inH);
480 return -1;
481 }
482 scaledOutH = scaledOutH & ~0x1; // make it multiple of 2
483
484 out->left = 0;
485 out->top = static_cast<int32_t>((inH - scaledOutH) / 2) & ~0x1;
486 out->width = static_cast<int32_t>(inW);
487 out->height = static_cast<int32_t>(scaledOutH);
488 ALOGV("%s: crop %dx%d to %dx%d: top %d, scaledH %d", __FUNCTION__, inW, inH, outW, outH,
489 out->top, static_cast<int32_t>(scaledOutH));
490 } else {
491 uint64_t scaledOutW = static_cast<uint64_t>(outW) * inH / outH;
492 if (scaledOutW > inW) {
493 ALOGE("%s: Output size %dx%d cannot be horizontally cropped from input size %dx%d",
494 __FUNCTION__, outW, outH, inW, inH);
495 return -1;
496 }
497 scaledOutW = scaledOutW & ~0x1; // make it multiple of 2
498
499 out->left = static_cast<int32_t>((inW - scaledOutW) / 2) & ~0x1;
500 out->top = 0;
501 out->width = static_cast<int32_t>(scaledOutW);
502 out->height = static_cast<int32_t>(inH);
503 ALOGV("%s: crop %dx%d to %dx%d: top %d, scaledW %d", __FUNCTION__, inW, inH, outW, outH,
504 out->top, static_cast<int32_t>(scaledOutW));
505 }
506
507 return 0;
508 }
509
formatConvert(const YCbCrLayout & in,const YCbCrLayout & out,Size sz,uint32_t format)510 int formatConvert(const YCbCrLayout& in, const YCbCrLayout& out, Size sz, uint32_t format) {
511 int ret = 0;
512 switch (format) {
513 case V4L2_PIX_FMT_NV21:
514 ret = libyuv::I420ToNV21(
515 static_cast<uint8_t*>(in.y), static_cast<int32_t>(in.yStride),
516 static_cast<uint8_t*>(in.cb), static_cast<int32_t>(in.cStride),
517 static_cast<uint8_t*>(in.cr), static_cast<int32_t>(in.cStride),
518 static_cast<uint8_t*>(out.y), static_cast<int32_t>(out.yStride),
519 static_cast<uint8_t*>(out.cr), static_cast<int32_t>(out.cStride),
520 static_cast<int32_t>(sz.width), static_cast<int32_t>(sz.height));
521 if (ret != 0) {
522 ALOGE("%s: convert to NV21 buffer failed! ret %d", __FUNCTION__, ret);
523 return ret;
524 }
525 break;
526 case V4L2_PIX_FMT_NV12:
527 ret = libyuv::I420ToNV12(
528 static_cast<uint8_t*>(in.y), static_cast<int32_t>(in.yStride),
529 static_cast<uint8_t*>(in.cb), static_cast<int32_t>(in.cStride),
530 static_cast<uint8_t*>(in.cr), static_cast<int32_t>(in.cStride),
531 static_cast<uint8_t*>(out.y), static_cast<int32_t>(out.yStride),
532 static_cast<uint8_t*>(out.cb), static_cast<int32_t>(out.cStride),
533 static_cast<int32_t>(sz.width), static_cast<int32_t>(sz.height));
534 if (ret != 0) {
535 ALOGE("%s: convert to NV12 buffer failed! ret %d", __FUNCTION__, ret);
536 return ret;
537 }
538 break;
539 case V4L2_PIX_FMT_YVU420: // YV12
540 case V4L2_PIX_FMT_YUV420: // YU12
541 // TODO: maybe we can speed up here by somehow save this copy?
542 ret = libyuv::I420Copy(static_cast<uint8_t*>(in.y), static_cast<int32_t>(in.yStride),
543 static_cast<uint8_t*>(in.cb), static_cast<int32_t>(in.cStride),
544 static_cast<uint8_t*>(in.cr), static_cast<int32_t>(in.cStride),
545 static_cast<uint8_t*>(out.y), static_cast<int32_t>(out.yStride),
546 static_cast<uint8_t*>(out.cb), static_cast<int32_t>(out.cStride),
547 static_cast<uint8_t*>(out.cr), static_cast<int32_t>(out.cStride),
548 static_cast<int32_t>(sz.width), static_cast<int32_t>(sz.height));
549 if (ret != 0) {
550 ALOGE("%s: copy to YV12 or YU12 buffer failed! ret %d", __FUNCTION__, ret);
551 return ret;
552 }
553 break;
554 case FLEX_YUV_GENERIC:
555 // TODO: b/72261744 write to arbitrary flexible YUV layout. Slow.
556 ALOGE("%s: unsupported flexible yuv layout"
557 " y %p cb %p cr %p y_str %d c_str %d c_step %d",
558 __FUNCTION__, out.y, out.cb, out.cr, out.yStride, out.cStride, out.chromaStep);
559 return -1;
560 default:
561 ALOGE("%s: unknown YUV format 0x%x!", __FUNCTION__, format);
562 return -1;
563 }
564 return 0;
565 }
566
encodeJpegYU12(const Size & inSz,const YCbCrLayout & inLayout,int jpegQuality,const void * app1Buffer,size_t app1Size,void * out,size_t maxOutSize,size_t & actualCodeSize)567 int encodeJpegYU12(const Size& inSz, const YCbCrLayout& inLayout, int jpegQuality,
568 const void* app1Buffer, size_t app1Size, void* out, size_t maxOutSize,
569 size_t& actualCodeSize) {
570 /* libjpeg is a C library so we use C-style "inheritance" by
571 * putting libjpeg's jpeg_destination_mgr first in our custom
572 * struct. This allows us to cast jpeg_destination_mgr* to
573 * CustomJpegDestMgr* when we get it passed to us in a callback */
574 struct CustomJpegDestMgr {
575 struct jpeg_destination_mgr mgr;
576 JOCTET* mBuffer;
577 size_t mBufferSize;
578 size_t mEncodedSize;
579 bool mSuccess;
580 } dmgr;
581
582 jpeg_compress_struct cinfo = {};
583 jpeg_error_mgr jerr;
584
585 /* Initialize error handling with standard callbacks, but
586 * then override output_message (to print to ALOG) and
587 * error_exit to set a flag and print a message instead
588 * of killing the whole process */
589 cinfo.err = jpeg_std_error(&jerr);
590
591 cinfo.err->output_message = [](j_common_ptr cinfo) {
592 char buffer[JMSG_LENGTH_MAX];
593
594 /* Create the message */
595 (*cinfo->err->format_message)(cinfo, buffer);
596 ALOGE("libjpeg error: %s", buffer);
597 };
598 cinfo.err->error_exit = [](j_common_ptr cinfo) {
599 (*cinfo->err->output_message)(cinfo);
600 if (cinfo->client_data) {
601 auto& dmgr = *reinterpret_cast<CustomJpegDestMgr*>(cinfo->client_data);
602 dmgr.mSuccess = false;
603 }
604 };
605
606 /* Now that we initialized some callbacks, let's create our compressor */
607 jpeg_create_compress(&cinfo);
608
609 /* Initialize our destination manager */
610 dmgr.mBuffer = static_cast<JOCTET*>(out);
611 dmgr.mBufferSize = maxOutSize;
612 dmgr.mEncodedSize = 0;
613 dmgr.mSuccess = true;
614 cinfo.client_data = static_cast<void*>(&dmgr);
615
616 /* These lambdas become C-style function pointers and as per C++11 spec
617 * may not capture anything */
618 dmgr.mgr.init_destination = [](j_compress_ptr cinfo) {
619 auto& dmgr = reinterpret_cast<CustomJpegDestMgr&>(*cinfo->dest);
620 dmgr.mgr.next_output_byte = dmgr.mBuffer;
621 dmgr.mgr.free_in_buffer = dmgr.mBufferSize;
622 ALOGV("%s:%d jpeg start: %p [%zu]", __FUNCTION__, __LINE__, dmgr.mBuffer, dmgr.mBufferSize);
623 };
624
625 dmgr.mgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) {
626 ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__);
627 return 0;
628 };
629
630 dmgr.mgr.term_destination = [](j_compress_ptr cinfo) {
631 auto& dmgr = reinterpret_cast<CustomJpegDestMgr&>(*cinfo->dest);
632 dmgr.mEncodedSize = dmgr.mBufferSize - dmgr.mgr.free_in_buffer;
633 ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, dmgr.mEncodedSize);
634 };
635 cinfo.dest = reinterpret_cast<struct jpeg_destination_mgr*>(&dmgr);
636
637 /* We are going to be using JPEG in raw data mode, so we are passing
638 * straight subsampled planar YCbCr and it will not touch our pixel
639 * data or do any scaling or anything */
640 cinfo.image_width = inSz.width;
641 cinfo.image_height = inSz.height;
642 cinfo.input_components = 3;
643 cinfo.in_color_space = JCS_YCbCr;
644
645 /* Initialize defaults and then override what we want */
646 jpeg_set_defaults(&cinfo);
647
648 jpeg_set_quality(&cinfo, jpegQuality, 1);
649 jpeg_set_colorspace(&cinfo, JCS_YCbCr);
650 cinfo.raw_data_in = 1;
651 cinfo.dct_method = JDCT_IFAST;
652
653 /* Configure sampling factors. The sampling factor is JPEG subsampling 420
654 * because the source format is YUV420. Note that libjpeg sampling factors
655 * are... a little weird. Sampling of Y=2,U=1,V=1 means there is 1 U and
656 * 1 V value for each 2 Y values */
657 cinfo.comp_info[0].h_samp_factor = 2;
658 cinfo.comp_info[0].v_samp_factor = 2;
659 cinfo.comp_info[1].h_samp_factor = 1;
660 cinfo.comp_info[1].v_samp_factor = 1;
661 cinfo.comp_info[2].h_samp_factor = 1;
662 cinfo.comp_info[2].v_samp_factor = 1;
663
664 /* Start the compressor */
665 jpeg_start_compress(&cinfo, TRUE);
666
667 /* Let's not hardcode YUV420 in 6 places... 5 was enough */
668 int maxVSampFactor = cinfo.max_v_samp_factor;
669 int cVSubSampling = cinfo.comp_info[0].v_samp_factor / cinfo.comp_info[1].v_samp_factor;
670
671 /* Compute our macroblock height, so we can pad our input to be vertically
672 * macroblock aligned. No need to for horizontal alignment since AllocatedFrame already
673 * pads horizontally */
674
675 size_t mcuV = DCTSIZE * maxVSampFactor;
676 size_t paddedHeight = mcuV * ((inSz.height + mcuV - 1) / mcuV);
677
678 /* libjpeg uses arrays of row pointers, which makes it really easy to pad
679 * data vertically (unfortunately doesn't help horizontally) */
680 std::vector<JSAMPROW> yLines(paddedHeight);
681 std::vector<JSAMPROW> cbLines(paddedHeight / cVSubSampling);
682 std::vector<JSAMPROW> crLines(paddedHeight / cVSubSampling);
683
684 uint8_t* py = static_cast<uint8_t*>(inLayout.y);
685 uint8_t* pcb = static_cast<uint8_t*>(inLayout.cb);
686 uint8_t* pcr = static_cast<uint8_t*>(inLayout.cr);
687
688 for (int32_t i = 0; i < paddedHeight; i++) {
689 /* Once we are in the padding territory we still point to the last line
690 * effectively replicating it several times ~ CLAMP_TO_EDGE */
691 int li = std::min(i, inSz.height - 1);
692 yLines[i] = static_cast<JSAMPROW>(py + li * inLayout.yStride);
693 if (i < paddedHeight / cVSubSampling) {
694 li = std::min(i, (inSz.height - 1) / cVSubSampling);
695 cbLines[i] = static_cast<JSAMPROW>(pcb + li * inLayout.cStride);
696 crLines[i] = static_cast<JSAMPROW>(pcr + li * inLayout.cStride);
697 }
698 }
699
700 /* If APP1 data was passed in, use it */
701 if (app1Buffer && app1Size) {
702 jpeg_write_marker(&cinfo, JPEG_APP0 + 1, static_cast<const JOCTET*>(app1Buffer), app1Size);
703 }
704
705 /* While we still have padded height left to go, keep giving it one
706 * macroblock at a time. */
707 while (cinfo.next_scanline < cinfo.image_height) {
708 const uint32_t batchSize = DCTSIZE * maxVSampFactor;
709 const uint32_t nl = cinfo.next_scanline;
710 JSAMPARRAY planes[3]{&yLines[nl], &cbLines[nl / cVSubSampling],
711 &crLines[nl / cVSubSampling]};
712
713 uint32_t done = jpeg_write_raw_data(&cinfo, planes, batchSize);
714
715 if (done != batchSize) {
716 ALOGE("%s: compressed %u lines, expected %u (total %u/%u)", __FUNCTION__, done,
717 batchSize, cinfo.next_scanline, cinfo.image_height);
718 return -1;
719 }
720 }
721
722 /* This will flush everything */
723 jpeg_finish_compress(&cinfo);
724
725 /* Grab the actual code size and set it */
726 actualCodeSize = dmgr.mEncodedSize;
727
728 return 0;
729 }
730
getMaxThumbnailResolution(const common::V1_0::helper::CameraMetadata & chars)731 Size getMaxThumbnailResolution(const common::V1_0::helper::CameraMetadata& chars) {
732 Size thumbSize{0, 0};
733 camera_metadata_ro_entry entry = chars.find(ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES);
734 for (uint32_t i = 0; i < entry.count; i += 2) {
735 Size sz{.width = entry.data.i32[i], .height = entry.data.i32[i + 1]};
736 if (sz.width * sz.height > thumbSize.width * thumbSize.height) {
737 thumbSize = sz;
738 }
739 }
740
741 if (thumbSize.width * thumbSize.height == 0) {
742 ALOGW("%s: non-zero thumbnail size not available", __FUNCTION__);
743 }
744
745 return thumbSize;
746 }
747
freeReleaseFences(std::vector<CaptureResult> & results)748 void freeReleaseFences(std::vector<CaptureResult>& results) {
749 for (auto& result : results) {
750 native_handle_t* inputReleaseFence =
751 ::android::makeFromAidl(result.inputBuffer.releaseFence);
752 if (inputReleaseFence != nullptr) {
753 native_handle_close(inputReleaseFence);
754 native_handle_delete(inputReleaseFence);
755 }
756 for (auto& buf : result.outputBuffers) {
757 native_handle_t* outReleaseFence = ::android::makeFromAidl(buf.releaseFence);
758 if (outReleaseFence != nullptr) {
759 native_handle_close(outReleaseFence);
760 native_handle_delete(outReleaseFence);
761 }
762 }
763 }
764 }
765
766 #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
767 #define UPDATE(md, tag, data, size) \
768 do { \
769 if ((md).update((tag), (data), (size))) { \
770 ALOGE("Update " #tag " failed!"); \
771 return BAD_VALUE; \
772 } \
773 } while (0)
774
fillCaptureResultCommon(CameraMetadata & md,nsecs_t timestamp,camera_metadata_ro_entry & activeArraySize)775 status_t fillCaptureResultCommon(CameraMetadata& md, nsecs_t timestamp,
776 camera_metadata_ro_entry& activeArraySize) {
777 if (activeArraySize.count < 4) {
778 ALOGE("%s: cannot find active array size!", __FUNCTION__);
779 return -EINVAL;
780 }
781 // android.control
782 // For USB camera, we don't know the AE state. Set the state to converged to
783 // indicate the frame should be good to use. Then apps don't have to wait the
784 // AE state.
785 const uint8_t aeState = ANDROID_CONTROL_AE_STATE_CONVERGED;
786 UPDATE(md, ANDROID_CONTROL_AE_STATE, &aeState, 1);
787
788 const uint8_t ae_lock = ANDROID_CONTROL_AE_LOCK_OFF;
789 UPDATE(md, ANDROID_CONTROL_AE_LOCK, &ae_lock, 1);
790
791 // Set AWB state to converged to indicate the frame should be good to use.
792 const uint8_t awbState = ANDROID_CONTROL_AWB_STATE_CONVERGED;
793 UPDATE(md, ANDROID_CONTROL_AWB_STATE, &awbState, 1);
794
795 const uint8_t awbLock = ANDROID_CONTROL_AWB_LOCK_OFF;
796 UPDATE(md, ANDROID_CONTROL_AWB_LOCK, &awbLock, 1);
797
798 const uint8_t flashState = ANDROID_FLASH_STATE_UNAVAILABLE;
799 UPDATE(md, ANDROID_FLASH_STATE, &flashState, 1);
800
801 // This means pipeline latency of X frame intervals. The maximum number is 4.
802 const uint8_t requestPipelineMaxDepth = 4;
803 UPDATE(md, ANDROID_REQUEST_PIPELINE_DEPTH, &requestPipelineMaxDepth, 1);
804
805 // android.scaler
806 const int32_t crop_region[] = {
807 activeArraySize.data.i32[0],
808 activeArraySize.data.i32[1],
809 activeArraySize.data.i32[2],
810 activeArraySize.data.i32[3],
811 };
812 UPDATE(md, ANDROID_SCALER_CROP_REGION, crop_region, ARRAY_SIZE(crop_region));
813
814 // android.sensor
815 UPDATE(md, ANDROID_SENSOR_TIMESTAMP, ×tamp, 1);
816
817 // android.statistics
818 const uint8_t lensShadingMapMode = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
819 UPDATE(md, ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, &lensShadingMapMode, 1);
820
821 const uint8_t sceneFlicker = ANDROID_STATISTICS_SCENE_FLICKER_NONE;
822 UPDATE(md, ANDROID_STATISTICS_SCENE_FLICKER, &sceneFlicker, 1);
823
824 return OK;
825 }
826
827 #undef ARRAY_SIZE
828 #undef UPDATE
829
AllocatedV4L2Frame(std::shared_ptr<V4L2Frame> frameIn)830 AllocatedV4L2Frame::AllocatedV4L2Frame(std::shared_ptr<V4L2Frame> frameIn)
831 : Frame(frameIn->mWidth, frameIn->mHeight, frameIn->mFourcc) {
832 uint8_t* dataIn;
833 size_t dataSize;
834 if (frameIn->getData(&dataIn, &dataSize) != 0) {
835 ALOGE("%s: map input V4L2 frame failed!", __FUNCTION__);
836 return;
837 }
838
839 mData.resize(dataSize);
840 std::memcpy(mData.data(), dataIn, dataSize);
841 }
842
~AllocatedV4L2Frame()843 AllocatedV4L2Frame::~AllocatedV4L2Frame() {}
844
getData(uint8_t ** outData,size_t * dataSize)845 int AllocatedV4L2Frame::getData(uint8_t** outData, size_t* dataSize) {
846 if (outData == nullptr || dataSize == nullptr) {
847 ALOGE("%s: outData(%p)/dataSize(%p) must not be null", __FUNCTION__, outData, dataSize);
848 return -1;
849 }
850
851 *outData = mData.data();
852 *dataSize = mData.size();
853 return 0;
854 }
855
856 } // namespace implementation
857 } // namespace device
858 } // namespace camera
859 } // namespace hardware
860 } // namespace android