1 /*
2 * cl_tnr_handler.cpp - CL tnr handler
3 *
4 * Copyright (c) 2015 Intel Corporation
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * Author: Wei Zong <wei.zong@intel.com>
19 */
20
21 #include "cl_tnr_handler.h"
22
23 #define TNR_PROCESSING_FRAME_COUNT 4
24 #define TNR_LIST_FRAME_COUNT 4
25 #define TNR_MOTION_THRESHOLD 2
26
27 namespace XCam {
28
29 static const XCamKernelInfo kernel_tnr_yuv_info = {
30 "kernel_tnr_yuv",
31 #include "kernel_tnr.clx"
32 , 0
33 };
34
35 static const XCamKernelInfo kernel_tnr_rgb_info = {
36 "kernel_tnr_rgb",
37 #include "kernel_tnr.clx"
38 , 0,
39 };
40
CLTnrMotionInfo()41 CLTnrImageHandler::CLTnrMotionInfo::CLTnrMotionInfo ()
42 : hor_shift (0)
43 , ver_shift (0)
44 , hor_corr (0)
45 , ver_corr (0)
46 {
47 }
48
CLTnrHistogram()49 CLTnrImageHandler::CLTnrHistogram::CLTnrHistogram() {
50 hor_hist_bin = 0;
51 ver_hist_bin = 0;
52 hor_hist_current = NULL;
53 hor_hist_reference = NULL;
54 ver_hist_current = NULL;
55 ver_hist_reference = NULL;
56 };
57
CLTnrHistogram(uint32_t width,uint32_t height)58 CLTnrImageHandler::CLTnrHistogram::CLTnrHistogram(uint32_t width, uint32_t height) {
59 hor_hist_bin = width;
60 ver_hist_bin = height;
61 if ((NULL == hor_hist_current) && (hor_hist_bin != 0)) {
62 hor_hist_current = (float*)xcam_malloc0(hor_hist_bin * sizeof(float));
63 }
64 if ((NULL == ver_hist_current) && (ver_hist_bin != 0)) {
65 ver_hist_current = (float*)xcam_malloc0(ver_hist_bin * sizeof(float));
66 }
67 if ((NULL == hor_hist_reference) && (hor_hist_bin != 0)) {
68 hor_hist_reference = (float*)xcam_malloc0(hor_hist_bin * sizeof(float));
69 }
70 if ((NULL == ver_hist_reference) && (ver_hist_bin != 0)) {
71 ver_hist_reference = (float*)xcam_malloc0(ver_hist_bin * sizeof(float));
72 }
73 };
74
~CLTnrHistogram()75 CLTnrImageHandler::CLTnrHistogram::~CLTnrHistogram() {
76 if (NULL != hor_hist_current) {
77 xcam_free(hor_hist_current);
78 hor_hist_current = NULL;
79 }
80 if (NULL != ver_hist_current) {
81 xcam_free(ver_hist_current);
82 ver_hist_current = NULL;
83 }
84 if (NULL != hor_hist_reference) {
85 xcam_free(hor_hist_reference);
86 hor_hist_reference = NULL;
87 }
88 if (NULL != ver_hist_reference) {
89 xcam_free(ver_hist_reference);
90 ver_hist_reference = NULL;
91 }
92 hor_hist_bin = 0;
93 ver_hist_bin = 0;
94 }
95
CLTnrImageKernel(const SmartPtr<CLContext> & context,CLTnrType type)96 CLTnrImageKernel::CLTnrImageKernel (
97 const SmartPtr<CLContext> &context, CLTnrType type)
98 : CLImageKernel (context)
99 , _type (type)
100 {
101 }
102
103 bool
calculate_image_histogram(XCam3AStats * stats,CLTnrHistogramType type,float * histogram)104 CLTnrImageHandler::calculate_image_histogram (XCam3AStats* stats, CLTnrHistogramType type, float* histogram)
105 {
106 if ( NULL == stats || NULL == histogram ) {
107 return false;
108 }
109
110 uint32_t normalize_factor = (1 << stats->info.bit_depth) - 1;
111 uint32_t image_width = stats->info.width;
112 uint32_t image_height = stats->info.height;
113 uint32_t image_aligned_width = stats->info.aligned_width;
114 uint32_t hor_hist_bin = image_width;
115 uint32_t ver_hist_bin = image_height;
116
117 switch (type) {
118 case CL_TNR_HIST_HOR_PROJECTION :
119 for (uint32_t bin = 0; bin < hor_hist_bin; bin++) {
120 for (uint32_t row_index = 0; row_index < image_height; row_index++) {
121 histogram[bin] += (float)(stats->stats[row_index * image_aligned_width + bin].avg_y)
122 / (1.0 * normalize_factor);
123 }
124 }
125 break;
126 case CL_TNR_HIST_VER_PROJECTION :
127 for (uint32_t bin = 0; bin < ver_hist_bin; bin++) {
128 for (uint32_t col_index = 0; col_index < image_width; col_index++) {
129 histogram[bin] += (float)(stats->stats[col_index + bin * image_aligned_width].avg_y)
130 / (1.0 * normalize_factor);
131 }
132 }
133 break;
134 case CL_TNR_HIST_BRIGHTNESS :
135 for (uint32_t row_index = 0; row_index < image_height; row_index++) {
136 for (uint32_t col_index = 0; col_index < image_width; col_index++) {
137 uint8_t bin = (stats->stats[row_index * image_aligned_width + col_index].avg_y * 255)
138 / normalize_factor;
139 histogram[bin]++;
140 }
141 }
142 break;
143 default :
144 break;
145 }
146
147 return true;
148 }
149
150 bool
calculate_image_histogram(SmartPtr<VideoBuffer> & input,CLTnrHistogramType type,float * histogram)151 CLTnrImageHandler::calculate_image_histogram (SmartPtr<VideoBuffer> &input, CLTnrHistogramType type, float* histogram)
152 {
153 if ( NULL == histogram ) {
154 return false;
155 }
156
157 uint32_t normalize_factor = (1 << input->get_video_info ().color_bits) - 1;
158 uint32_t image_width = input->get_video_info ().width;
159 uint32_t image_height = input->get_video_info ().height;
160 uint32_t image_aligned_width = input->get_video_info ().aligned_width;
161 uint32_t stride = input->get_video_info ().strides[0];
162
163 uint32_t hor_hist_bin = image_width;
164 uint32_t ver_hist_bin = image_height;
165 uint32_t pxiel_bytes = stride / image_aligned_width;
166
167 uint32_t format = input->get_video_info ().format;
168 if (XCAM_PIX_FMT_RGBA64 != format) {
169 XCAM_LOG_ERROR ("Only support RGBA64 format !");
170 return false;
171 }
172
173 uint8_t* image_buffer = input->map();
174 if (NULL == image_buffer) {
175 return false;
176 }
177
178 switch (type) {
179 case CL_TNR_HIST_HOR_PROJECTION :
180 for (uint32_t bin = 0; bin < hor_hist_bin; bin++) {
181 for (uint32_t row_index = 0; row_index < image_height; row_index++) {
182 histogram[bin] += (float)(image_buffer[row_index * stride + pxiel_bytes * bin] +
183 (image_buffer[row_index * stride + pxiel_bytes * bin + 1] << 8) +
184 image_buffer[row_index * stride + pxiel_bytes * bin + 2] +
185 (image_buffer[row_index * stride + pxiel_bytes * bin + 3] << 8) +
186 image_buffer[row_index * stride + pxiel_bytes * bin + 4] +
187 (image_buffer[row_index * stride + pxiel_bytes * bin + 5] << 8) )
188 / (3.0 * normalize_factor);
189 }
190 }
191 break;
192 case CL_TNR_HIST_VER_PROJECTION :
193 for (uint32_t bin = 0; bin < ver_hist_bin; bin++) {
194 for (uint32_t col_index = 0; col_index < stride; col_index += pxiel_bytes) {
195 histogram[bin] += (float)(image_buffer[col_index + bin * stride] +
196 (image_buffer[col_index + bin * stride + 1] << 8) +
197 image_buffer[col_index + bin * stride + 2] +
198 (image_buffer[col_index + bin * stride + 3] << 8) +
199 image_buffer[col_index + bin * stride + 4] +
200 (image_buffer[col_index + bin * stride + 5] << 8) )
201 / (3.0 * normalize_factor);
202 }
203 }
204 break;
205 case CL_TNR_HIST_BRIGHTNESS :
206 for (uint32_t row_index = 0; row_index < image_height; row_index++) {
207 for (uint32_t col_index = 0; col_index < stride; col_index += pxiel_bytes) {
208 uint8_t bin = (image_buffer[row_index * stride + col_index] +
209 (image_buffer[row_index * stride + col_index + 1] << 8) +
210 image_buffer[row_index * stride + col_index + 2] +
211 (image_buffer[row_index * stride + col_index + 3] << 8) +
212 image_buffer[row_index * stride + col_index + 4] +
213 (image_buffer[row_index * stride + col_index + 5] << 8) ) * 255
214 / (3 * normalize_factor);
215 histogram[bin]++;
216 }
217 }
218 break;
219 default :
220 break;
221 }
222
223 input->unmap();
224
225 return true;
226 }
227
228 void
print_image_histogram()229 CLTnrImageHandler::print_image_histogram ()
230 {
231 uint32_t hor_hist_bin = _image_histogram.hor_hist_bin;
232 uint32_t ver_hist_bin = _image_histogram.ver_hist_bin;
233
234 XCAM_LOG_DEBUG ("hor hist bin = %d, ver hist bin = %d", hor_hist_bin, ver_hist_bin);
235
236 printf("float hor_hist_current[] = { ");
237 for (uint32_t i = 0; i < hor_hist_bin; i++) {
238 printf("%f, ", _image_histogram.hor_hist_current[i]);
239 }
240 printf(" }; \n\n\n");
241
242 printf("float ver_hist_current[] = { ");
243 for (uint32_t i = 0; i < ver_hist_bin; i++) {
244 printf("%f, ", _image_histogram.ver_hist_current[i]);
245 }
246 printf(" }; \n\n\n");
247
248 printf("float hor_hist_reference[] = { ");
249 for (uint32_t i = 0; i < hor_hist_bin; i++) {
250 printf("%f, ", _image_histogram.hor_hist_reference[i]);
251 }
252 printf(" }; \n\n\n");
253
254 printf("float ver_hist_reference[] = { ");
255 for (uint32_t i = 0; i < ver_hist_bin; i++) {
256 printf("%f, ", _image_histogram.ver_hist_reference[i]);
257 }
258 printf(" }; \n\n\n");
259 }
260
CLTnrImageHandler(const SmartPtr<CLContext> & context,CLTnrType type,const char * name)261 CLTnrImageHandler::CLTnrImageHandler (const SmartPtr<CLContext> &context, CLTnrType type, const char *name)
262 : CLImageHandler (context, name)
263 , _type (type)
264 , _gain_yuv (1.0)
265 , _thr_y (0.05)
266 , _thr_uv (0.05)
267 , _gain_rgb (0.0)
268 , _thr_r (0.064) // set high initial threshold to get strong denoise effect
269 , _thr_g (0.045)
270 , _thr_b (0.073)
271 , _frame_count (TNR_PROCESSING_FRAME_COUNT)
272 {
273 }
274
275 bool
set_tnr_kernel(SmartPtr<CLTnrImageKernel> & kernel)276 CLTnrImageHandler::set_tnr_kernel(SmartPtr<CLTnrImageKernel> &kernel)
277 {
278 SmartPtr<CLImageKernel> image_kernel = kernel;
279 add_kernel (image_kernel);
280 _tnr_kernel = kernel;
281 return true;
282 }
283
284 bool
set_framecount(uint8_t count)285 CLTnrImageHandler::set_framecount (uint8_t count)
286 {
287 if (!_tnr_kernel->is_valid ()) {
288 XCAM_LOG_ERROR ("set framecount error, invalid TNR kernel !");
289 return false;
290 }
291
292 XCAM_ASSERT (count >= 2 && count <= 4);
293 _frame_count = count;
294
295 return true;
296 }
297
298 bool
set_rgb_config(const XCam3aResultTemporalNoiseReduction & config)299 CLTnrImageHandler::set_rgb_config (const XCam3aResultTemporalNoiseReduction& config)
300
301 {
302 if (!_tnr_kernel->is_valid ()) {
303 XCAM_LOG_ERROR ("set threshold error, invalid TNR kernel !");
304 return false;
305 }
306 _gain_rgb = (float)config.gain;
307 _thr_r = (float)config.threshold[0];
308 _thr_g = (float)config.threshold[1];
309 _thr_b = (float)config.threshold[2];
310 XCAM_LOG_DEBUG ("set TNR RGB config: _gain(%f), _thr_r(%f), _thr_g(%f), _thr_b(%f)",
311 _gain_rgb, _thr_r, _thr_g, _thr_b);
312
313 return true;
314 }
315
316 bool
set_yuv_config(const XCam3aResultTemporalNoiseReduction & config)317 CLTnrImageHandler::set_yuv_config (const XCam3aResultTemporalNoiseReduction& config)
318
319 {
320 if (!_tnr_kernel->is_valid ()) {
321 XCAM_LOG_ERROR ("set threshold error, invalid TNR kernel !");
322 return false;
323 }
324
325 _gain_yuv = (float)config.gain;
326 _thr_y = (float)config.threshold[0];
327 _thr_uv = (float)config.threshold[1];
328
329 XCAM_LOG_DEBUG ("set TNR YUV config: _gain(%f), _thr_y(%f), _thr_uv(%f)",
330 _gain_yuv, _thr_y, _thr_uv);
331
332 return true;
333 }
334
335 XCamReturn
prepare_parameters(SmartPtr<VideoBuffer> & input,SmartPtr<VideoBuffer> & output)336 CLTnrImageHandler::prepare_parameters (SmartPtr<VideoBuffer> &input, SmartPtr<VideoBuffer> &output)
337 {
338 SmartPtr<CLContext> context = get_context ();
339 const VideoBufferInfo & video_info = input->get_video_info ();
340 CLArgList args;
341 CLWorkSize work_size;
342 XCamReturn ret = XCAM_RETURN_NO_ERROR;
343
344 XCAM_ASSERT (_tnr_kernel.ptr ());
345
346 CLImageDesc desc;
347 if (CL_TNR_TYPE_YUV == _type) {
348 desc.format.image_channel_order = CL_R;
349 desc.format.image_channel_data_type = CL_UNORM_INT8;
350 desc.width = video_info.aligned_width;
351 desc.height = video_info.aligned_height + video_info.height / 2;
352 desc.row_pitch = video_info.strides[0];
353 desc.array_size = 2;
354 desc.slice_pitch = video_info.strides [0] * video_info.aligned_height;
355 } else if (CL_TNR_TYPE_RGB == _type) {
356 desc.format.image_channel_order = CL_RGBA;
357 desc.format.image_channel_data_type = CL_UNORM_INT8;
358 desc.width = video_info.aligned_width;
359 desc.height = video_info.height;
360 desc.row_pitch = video_info.strides[0];
361 desc.array_size = 0;
362 desc.slice_pitch = 0;
363 }
364
365 SmartPtr<CLImage> image_in = convert_to_climage (context, input, desc);
366 SmartPtr<CLImage> image_out = convert_to_climage (context, output, desc);
367
368 XCAM_FAIL_RETURN (
369 WARNING,
370 image_in->is_valid () && image_out->is_valid (),
371 XCAM_RETURN_ERROR_MEM,
372 "cl image kernel(%s) in/out memory not available", _tnr_kernel->get_kernel_name ());
373
374 if (CL_TNR_TYPE_YUV == _type) {
375 if (!_image_out_prev.ptr ()) {
376 _image_out_prev = image_in;
377 }
378 } else if (CL_TNR_TYPE_RGB == _type) {
379 // analyze motion between the latest adjacent two frames
380 // Todo: enable analyze when utilize motion compensation next step
381
382 if (_image_in_list.size () < TNR_LIST_FRAME_COUNT) {
383 while (_image_in_list.size () < TNR_LIST_FRAME_COUNT) {
384 _image_in_list.push_back (image_in);
385 }
386 } else {
387 _image_in_list.pop_front ();
388 _image_in_list.push_back (image_in);
389 }
390 }
391
392 uint32_t vertical_offset = video_info.aligned_height;
393
394 //set args;
395 work_size.dim = XCAM_DEFAULT_IMAGE_DIM;
396 work_size.local[0] = 8;
397 work_size.local[1] = 4;
398 if (CL_TNR_TYPE_YUV == _type) {
399 args.push_back (new CLMemArgument (image_in));
400 args.push_back (new CLMemArgument (_image_out_prev));
401 args.push_back (new CLMemArgument (image_out));
402 args.push_back (new CLArgumentT<uint> (vertical_offset));
403
404 args.push_back (new CLArgumentT<float> (_gain_yuv));
405 args.push_back (new CLArgumentT<float> (_thr_y));
406 args.push_back (new CLArgumentT<float> (_thr_uv));
407
408 work_size.global[0] = video_info.width / 2;
409 work_size.global[1] = video_info.height / 2;
410 }
411 else if (CL_TNR_TYPE_RGB == _type) {
412 const CLImageDesc out_info = image_out->get_image_desc ();
413 work_size.global[0] = out_info.width;
414 work_size.global[1] = out_info.height;
415
416 args.push_back (new CLMemArgument (image_out));
417 args.push_back (new CLArgumentT<float> (_gain_rgb));
418 args.push_back (new CLArgumentT<float> (_thr_r));
419 args.push_back (new CLArgumentT<float> (_thr_g));
420 args.push_back (new CLArgumentT<float> (_thr_b));
421 args.push_back (new CLArgumentT<uint8_t> (_frame_count));
422
423 for (std::list<SmartPtr<CLImage>>::iterator it = _image_in_list.begin (); it != _image_in_list.end (); it++) {
424 args.push_back (new CLMemArgument (*it));
425 }
426 }
427
428 XCAM_ASSERT (_tnr_kernel.ptr ());
429 ret = _tnr_kernel->set_arguments (args, work_size);
430 XCAM_FAIL_RETURN (
431 WARNING, ret == XCAM_RETURN_NO_ERROR, ret,
432 "tnr kernel set arguments failed.");
433
434 _image_out_prev = image_out;
435 return XCAM_RETURN_NO_ERROR;
436 }
437
438 SmartPtr<CLImageHandler>
create_cl_tnr_image_handler(const SmartPtr<CLContext> & context,CLTnrType type)439 create_cl_tnr_image_handler (const SmartPtr<CLContext> &context, CLTnrType type)
440 {
441 SmartPtr<CLTnrImageHandler> tnr_handler;
442 SmartPtr<CLTnrImageKernel> tnr_kernel;
443 XCamReturn ret = XCAM_RETURN_NO_ERROR;
444
445 tnr_kernel = new CLTnrImageKernel (context, type);
446 XCAM_ASSERT (tnr_kernel.ptr ());
447 if (CL_TNR_TYPE_YUV == type) {
448 ret = tnr_kernel->build_kernel (kernel_tnr_yuv_info, NULL);
449 } else if (CL_TNR_TYPE_RGB == type) {
450 ret = tnr_kernel->build_kernel (kernel_tnr_rgb_info, NULL);
451 } else {
452 XCAM_LOG_ERROR ("create cl tnr image handler failed, unknown type:%d", type);
453 return NULL;
454 }
455
456 XCAM_FAIL_RETURN (
457 ERROR, ret == XCAM_RETURN_NO_ERROR, NULL,
458 "build tnr kernel failed");
459
460 tnr_handler = new CLTnrImageHandler (context, type, "cl_handler_tnr");
461 XCAM_ASSERT (tnr_kernel->is_valid ());
462 tnr_handler->set_tnr_kernel (tnr_kernel);
463
464 return tnr_handler;
465 }
466
467 };
468