1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "JpegCompressor"
19
20 #include "JpegCompressor.h"
21
22 #include <camera_blob.h>
23 #include <cutils/properties.h>
24 #include <libyuv.h>
25 #include <utils/Log.h>
26 #include <utils/Trace.h>
27
28 namespace android {
29
30 using google_camera_hal::CameraBlob;
31 using google_camera_hal::CameraBlobId;
32 using google_camera_hal::ErrorCode;
33 using google_camera_hal::MessageType;
34 using google_camera_hal::NotifyMessage;
35
JpegCompressor()36 JpegCompressor::JpegCompressor() {
37 ATRACE_CALL();
38 char value[PROPERTY_VALUE_MAX];
39 if (property_get("ro.product.manufacturer", value, "unknown") <= 0) {
40 ALOGW("%s: No Exif make data!", __FUNCTION__);
41 }
42 exif_make_ = std::string(value);
43
44 if (property_get("ro.product.model", value, "unknown") <= 0) {
45 ALOGW("%s: No Exif model data!", __FUNCTION__);
46 }
47 exif_model_ = std::string(value);
48
49 jpeg_processing_thread_ = std::thread([this] { this->ThreadLoop(); });
50 }
51
~JpegCompressor()52 JpegCompressor::~JpegCompressor() {
53 ATRACE_CALL();
54
55 // Abort the ongoing compression and flush any pending jobs
56 jpeg_done_ = true;
57 condition_.notify_one();
58 jpeg_processing_thread_.join();
59 while (!pending_yuv_jobs_.empty()) {
60 auto job = std::move(pending_yuv_jobs_.front());
61 job->output->stream_buffer.status = BufferStatus::kError;
62 pending_yuv_jobs_.pop();
63 }
64 }
65
QueueYUV420(std::unique_ptr<JpegYUV420Job> job)66 status_t JpegCompressor::QueueYUV420(std::unique_ptr<JpegYUV420Job> job) {
67 ATRACE_CALL();
68
69 if ((job->input.get() == nullptr) || (job->output.get() == nullptr) ||
70 (job->output->format != PixelFormat::BLOB) ||
71 (job->output->dataSpace != HAL_DATASPACE_V0_JFIF)) {
72 ALOGE("%s: Unable to find buffers for JPEG source/destination",
73 __FUNCTION__);
74 return BAD_VALUE;
75 }
76
77 std::unique_lock<std::mutex> lock(mutex_);
78 pending_yuv_jobs_.push(std::move(job));
79 condition_.notify_one();
80
81 return OK;
82 }
83
ThreadLoop()84 void JpegCompressor::ThreadLoop() {
85 ATRACE_CALL();
86
87 while (!jpeg_done_) {
88 std::unique_ptr<JpegYUV420Job> current_yuv_job = nullptr;
89 {
90 std::lock_guard<std::mutex> lock(mutex_);
91 if (!pending_yuv_jobs_.empty()) {
92 current_yuv_job = std::move(pending_yuv_jobs_.front());
93 pending_yuv_jobs_.pop();
94 }
95 }
96
97 if (current_yuv_job.get() != nullptr) {
98 CompressYUV420(std::move(current_yuv_job));
99 }
100
101 std::unique_lock<std::mutex> lock(mutex_);
102 auto ret = condition_.wait_for(lock, std::chrono::milliseconds(10));
103 if (ret == std::cv_status::timeout) {
104 ALOGV("%s: Jpeg thread timeout", __FUNCTION__);
105 }
106 }
107 }
108
CompressYUV420(std::unique_ptr<JpegYUV420Job> job)109 void JpegCompressor::CompressYUV420(std::unique_ptr<JpegYUV420Job> job) {
110 const uint8_t* app1_buffer = nullptr;
111 size_t app1_buffer_size = 0;
112 std::vector<uint8_t> thumbnail_jpeg_buffer;
113 size_t encoded_thumbnail_size = 0;
114 if ((job->exif_utils.get() != nullptr) &&
115 (job->result_metadata.get() != nullptr)) {
116 if (job->exif_utils->Initialize()) {
117 camera_metadata_ro_entry_t entry;
118 size_t thumbnail_width = 0;
119 size_t thumbnail_height = 0;
120 std::vector<uint8_t> thumb_yuv420_frame;
121 YCbCrPlanes thumb_planes;
122 auto ret = job->result_metadata->Get(ANDROID_JPEG_THUMBNAIL_SIZE, &entry);
123 if ((ret == OK) && (entry.count == 2)) {
124 thumbnail_width = entry.data.i32[0];
125 thumbnail_height = entry.data.i32[1];
126 if ((thumbnail_width > 0) && (thumbnail_height > 0)) {
127 thumb_yuv420_frame.resize((thumbnail_width * thumbnail_height * 3) /
128 2);
129 thumb_planes = {
130 .img_y = thumb_yuv420_frame.data(),
131 .img_cb = thumb_yuv420_frame.data() +
132 thumbnail_width * thumbnail_height,
133 .img_cr = thumb_yuv420_frame.data() +
134 (thumbnail_width * thumbnail_height * 5) / 4,
135 .y_stride = static_cast<uint32_t>(thumbnail_width),
136 .cbcr_stride = static_cast<uint32_t>(thumbnail_width) / 2};
137 // TODO: Crop thumbnail according to documentation
138 auto stat = I420Scale(
139 job->input->yuv_planes.img_y, job->input->yuv_planes.y_stride,
140 job->input->yuv_planes.img_cb, job->input->yuv_planes.cbcr_stride,
141 job->input->yuv_planes.img_cr, job->input->yuv_planes.cbcr_stride,
142 job->input->width, job->input->height, thumb_planes.img_y,
143 thumb_planes.y_stride, thumb_planes.img_cb,
144 thumb_planes.cbcr_stride, thumb_planes.img_cr,
145 thumb_planes.cbcr_stride, thumbnail_width, thumbnail_height,
146 libyuv::kFilterNone);
147 if (stat != 0) {
148 ALOGE("%s: Failed during thumbnail scaling: %d", __FUNCTION__, stat);
149 thumb_yuv420_frame.clear();
150 }
151 }
152 }
153
154 if (job->exif_utils->SetFromMetadata(
155 *job->result_metadata, job->input->width, job->input->height)) {
156 if (!thumb_yuv420_frame.empty()) {
157 thumbnail_jpeg_buffer.resize(64 * 1024); // APP1 is limited by 64k
158 encoded_thumbnail_size = CompressYUV420Frame(
159 {.output_buffer = thumbnail_jpeg_buffer.data(),
160 .output_buffer_size = thumbnail_jpeg_buffer.size(),
161 .yuv_planes = thumb_planes,
162 .width = thumbnail_width,
163 .height = thumbnail_height,
164 .app1_buffer = nullptr,
165 .app1_buffer_size = 0});
166 if (encoded_thumbnail_size > 0) {
167 job->output->stream_buffer.status = BufferStatus::kOk;
168 } else {
169 ALOGE("%s: Failed encoding thumbail!", __FUNCTION__);
170 thumbnail_jpeg_buffer.clear();
171 }
172 }
173
174 job->exif_utils->SetMake(exif_make_);
175 job->exif_utils->SetModel(exif_model_);
176 if (job->exif_utils->GenerateApp1(thumbnail_jpeg_buffer.empty()
177 ? nullptr
178 : thumbnail_jpeg_buffer.data(),
179 encoded_thumbnail_size)) {
180 app1_buffer = job->exif_utils->GetApp1Buffer();
181 app1_buffer_size = job->exif_utils->GetApp1Length();
182 } else {
183 ALOGE("%s: Unable to generate App1 buffer", __FUNCTION__);
184 }
185 } else {
186 ALOGE("%s: Unable to generate EXIF section!", __FUNCTION__);
187 }
188 } else {
189 ALOGE("%s: Unable to initialize Exif generator!", __FUNCTION__);
190 }
191 }
192
193 auto encoded_size = CompressYUV420Frame(
194 {.output_buffer = job->output->plane.img.img,
195 .output_buffer_size = job->output->plane.img.buffer_size,
196 .yuv_planes = job->input->yuv_planes,
197 .width = job->input->width,
198 .height = job->input->height,
199 .app1_buffer = app1_buffer,
200 .app1_buffer_size = app1_buffer_size});
201 if (encoded_size > 0) {
202 job->output->stream_buffer.status = BufferStatus::kOk;
203 } else {
204 job->output->stream_buffer.status = BufferStatus::kError;
205 return;
206 }
207
208 auto jpeg_header_offset =
209 job->output->plane.img.buffer_size - sizeof(struct CameraBlob);
210 if (jpeg_header_offset > encoded_size) {
211 struct CameraBlob* blob = reinterpret_cast<struct CameraBlob*>(
212 job->output->plane.img.img + jpeg_header_offset);
213 blob->blob_id = CameraBlobId::JPEG;
214 blob->blob_size = encoded_size;
215 } else {
216 ALOGW("%s: No space for jpeg header at offset: %u and jpeg size: %u",
217 __FUNCTION__, static_cast<unsigned>(jpeg_header_offset),
218 static_cast<unsigned>(encoded_size));
219 }
220 }
221
CompressYUV420Frame(YUV420Frame frame)222 size_t JpegCompressor::CompressYUV420Frame(YUV420Frame frame) {
223 ATRACE_CALL();
224
225 struct CustomJpegDestMgr : public jpeg_destination_mgr {
226 JOCTET* buffer;
227 size_t buffer_size;
228 size_t encoded_size;
229 bool success;
230 } dmgr;
231
232 // Set up error management
233 jpeg_error_info_ = NULL;
234 jpeg_error_mgr jerr;
235
236 auto cinfo = std::make_unique<jpeg_compress_struct>();
237 cinfo->err = jpeg_std_error(&jerr);
238 cinfo->err->error_exit = [](j_common_ptr cinfo) {
239 (*cinfo->err->output_message)(cinfo);
240 if (cinfo->client_data) {
241 auto& dmgr = *static_cast<CustomJpegDestMgr*>(cinfo->client_data);
242 dmgr.success = false;
243 }
244 };
245
246 jpeg_create_compress(cinfo.get());
247 if (CheckError("Error initializing compression")) {
248 return 0;
249 }
250
251 dmgr.buffer = static_cast<JOCTET*>(frame.output_buffer);
252 dmgr.buffer_size = frame.output_buffer_size;
253 dmgr.encoded_size = 0;
254 dmgr.success = true;
255 cinfo->client_data = static_cast<void*>(&dmgr);
256 dmgr.init_destination = [](j_compress_ptr cinfo) {
257 auto& dmgr = static_cast<CustomJpegDestMgr&>(*cinfo->dest);
258 dmgr.next_output_byte = dmgr.buffer;
259 dmgr.free_in_buffer = dmgr.buffer_size;
260 ALOGV("%s:%d jpeg start: %p [%zu]", __FUNCTION__, __LINE__, dmgr.buffer,
261 dmgr.buffer_size);
262 };
263
264 dmgr.empty_output_buffer = [](j_compress_ptr) {
265 ALOGE("%s:%d Out of buffer", __FUNCTION__, __LINE__);
266 return 0;
267 };
268
269 dmgr.term_destination = [](j_compress_ptr cinfo) {
270 auto& dmgr = static_cast<CustomJpegDestMgr&>(*cinfo->dest);
271 dmgr.encoded_size = dmgr.buffer_size - dmgr.free_in_buffer;
272 ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__,
273 dmgr.encoded_size);
274 };
275
276 cinfo->dest = reinterpret_cast<struct jpeg_destination_mgr*>(&dmgr);
277
278 // Set up compression parameters
279 cinfo->image_width = frame.width;
280 cinfo->image_height = frame.height;
281 cinfo->input_components = 3;
282 cinfo->in_color_space = JCS_YCbCr;
283
284 jpeg_set_defaults(cinfo.get());
285 if (CheckError("Error configuring defaults")) {
286 return 0;
287 }
288
289 jpeg_set_colorspace(cinfo.get(), JCS_YCbCr);
290 if (CheckError("Error configuring color space")) {
291 return 0;
292 }
293
294 cinfo->raw_data_in = 1;
295 // YUV420 planar with chroma subsampling
296 cinfo->comp_info[0].h_samp_factor = 2;
297 cinfo->comp_info[0].v_samp_factor = 2;
298 cinfo->comp_info[1].h_samp_factor = 1;
299 cinfo->comp_info[1].v_samp_factor = 1;
300 cinfo->comp_info[2].h_samp_factor = 1;
301 cinfo->comp_info[2].v_samp_factor = 1;
302
303 int max_vsamp_factor = std::max({cinfo->comp_info[0].v_samp_factor,
304 cinfo->comp_info[1].v_samp_factor,
305 cinfo->comp_info[2].v_samp_factor});
306 int c_vsub_sampling =
307 cinfo->comp_info[0].v_samp_factor / cinfo->comp_info[1].v_samp_factor;
308
309 // Start compression
310 jpeg_start_compress(cinfo.get(), TRUE);
311 if (CheckError("Error starting compression")) {
312 return 0;
313 }
314
315 if ((frame.app1_buffer != nullptr) && (frame.app1_buffer_size > 0)) {
316 jpeg_write_marker(cinfo.get(), JPEG_APP0 + 1,
317 static_cast<const JOCTET*>(frame.app1_buffer),
318 frame.app1_buffer_size);
319 }
320
321 // Compute our macroblock height, so we can pad our input to be vertically
322 // macroblock aligned.
323
324 size_t mcu_v = DCTSIZE * max_vsamp_factor;
325 size_t padded_height = mcu_v * ((cinfo->image_height + mcu_v - 1) / mcu_v);
326
327 std::vector<JSAMPROW> y_lines(padded_height);
328 std::vector<JSAMPROW> cb_lines(padded_height / c_vsub_sampling);
329 std::vector<JSAMPROW> cr_lines(padded_height / c_vsub_sampling);
330
331 uint8_t* py = static_cast<uint8_t*>(frame.yuv_planes.img_y);
332 uint8_t* pcr = static_cast<uint8_t*>(frame.yuv_planes.img_cr);
333 uint8_t* pcb = static_cast<uint8_t*>(frame.yuv_planes.img_cb);
334
335 for (uint32_t i = 0; i < padded_height; i++) {
336 /* Once we are in the padding territory we still point to the last line
337 * effectively replicating it several times ~ CLAMP_TO_EDGE */
338 int li = std::min(i, cinfo->image_height - 1);
339 y_lines[i] = static_cast<JSAMPROW>(py + li * frame.yuv_planes.y_stride);
340 if (i < padded_height / c_vsub_sampling) {
341 li = std::min(i, (cinfo->image_height - 1) / c_vsub_sampling);
342 cr_lines[i] =
343 static_cast<JSAMPROW>(pcr + li * frame.yuv_planes.cbcr_stride);
344 cb_lines[i] =
345 static_cast<JSAMPROW>(pcb + li * frame.yuv_planes.cbcr_stride);
346 }
347 }
348
349 const uint32_t batch_size = DCTSIZE * max_vsamp_factor;
350 while (cinfo->next_scanline < cinfo->image_height) {
351 JSAMPARRAY planes[3]{&y_lines[cinfo->next_scanline],
352 &cb_lines[cinfo->next_scanline / c_vsub_sampling],
353 &cr_lines[cinfo->next_scanline / c_vsub_sampling]};
354
355 jpeg_write_raw_data(cinfo.get(), planes, batch_size);
356 if (CheckError("Error while compressing")) {
357 return 0;
358 }
359
360 if (jpeg_done_) {
361 ALOGV("%s: Cancel called, exiting early", __FUNCTION__);
362 jpeg_finish_compress(cinfo.get());
363 return 0;
364 }
365 }
366
367 jpeg_finish_compress(cinfo.get());
368 if (CheckError("Error while finishing compression")) {
369 return 0;
370 }
371
372 return dmgr.encoded_size;
373 }
374
CheckError(const char * msg)375 bool JpegCompressor::CheckError(const char* msg) {
376 if (jpeg_error_info_) {
377 char err_buffer[JMSG_LENGTH_MAX];
378 jpeg_error_info_->err->format_message(jpeg_error_info_, err_buffer);
379 ALOGE("%s: %s: %s", __FUNCTION__, msg, err_buffer);
380 jpeg_error_info_ = NULL;
381 return true;
382 }
383
384 return false;
385 }
386
387 } // namespace android
388