1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Image Field-of-View utilities for aspect ratio, crop, and FoV tests.""" 15 16 17import logging 18import math 19 20import cv2 21import numpy as np 22 23import camera_properties_utils 24import capture_request_utils 25import image_processing_utils 26import opencv_processing_utils 27 28CIRCLE_COLOR = 0 # [0: black, 255: white] 29CIRCLE_MIN_AREA = 0.01 # 1% of image size 30FOV_PERCENT_RTOL = 0.15 # Relative tolerance on circle FoV % to expected. 31LARGE_SIZE_IMAGE = 2000 # Size of a large image (compared against max(w, h)) 32THRESH_AR_L = 0.02 # Aspect ratio test threshold of large images 33THRESH_AR_S = 0.075 # Aspect ratio test threshold of mini images 34THRESH_CROP_L = 0.02 # Crop test threshold of large images 35THRESH_CROP_S = 0.075 # Crop test threshold of mini images 36THRESH_MIN_PIXEL = 4 # Crop test allowed offset 37 38 39def calc_camera_fov_from_metadata(metadata, props): 40 """Get field of view of camera. 41 42 Args: 43 metadata: capture result's metadata. 44 props: obj; camera properties object. 45 Returns: 46 fov: int; field of view of camera. 47 """ 48 sensor_size = props['android.sensor.info.physicalSize'] 49 diag = math.sqrt(sensor_size['height']**2 + sensor_size['width']**2) 50 fl = metadata['android.lens.focalLength'] 51 logging.debug('Focal length: %.3f', fl) 52 fov = 2 * math.degrees(math.atan(diag / (2 * fl))) 53 logging.debug('Field of view: %.1f degrees', fov) 54 return fov 55 56 57def calc_scaler_crop_region_ratio(scaler_crop_region, props): 58 """Calculate ratio of scaler crop region area over active array area. 59 60 Args: 61 scaler_crop_region: Rect(left, top, right, bottom) 62 props: camera properties 63 64 Returns: 65 ratio of scaler crop region area over active array area 66 """ 67 a = props['android.sensor.info.activeArraySize'] 68 s = scaler_crop_region 69 logging.debug('Active array size: %s', a) 70 active_array_area = (a['right'] - a['left']) * (a['bottom'] - a['top']) 71 scaler_crop_region_area = (s['right'] - s['left']) * (s['bottom'] - s['top']) 72 crop_region_active_array_ratio = scaler_crop_region_area / active_array_area 73 return crop_region_active_array_ratio 74 75 76def check_fov(circle, ref_fov, w, h): 77 """Check the FoV for correct size.""" 78 fov_percent = calc_circle_image_ratio(circle['r'], w, h) 79 chk_percent = calc_expected_circle_image_ratio(ref_fov, w, h) 80 if not math.isclose(fov_percent, chk_percent, rel_tol=FOV_PERCENT_RTOL): 81 e_msg = (f'FoV %: {fov_percent:.2f}, Ref FoV %: {chk_percent:.2f}, ' 82 f'TOL={FOV_PERCENT_RTOL*100}%, img: {w}x{h}, ref: ' 83 f"{ref_fov['w']}x{ref_fov['h']}") 84 return e_msg 85 86 87def check_ar(circle, ar_gt, w, h, e_msg_stem): 88 """Check the aspect ratio of the circle. 89 90 size is the larger of w or h. 91 if size >= LARGE_SIZE_IMAGE: use THRESH_AR_L 92 elif size == 0 (extreme case): THRESH_AR_S 93 elif 0 < image size < LARGE_SIZE_IMAGE: scale between THRESH_AR_S & AR_L 94 95 Args: 96 circle: dict with circle parameters 97 ar_gt: aspect ratio ground truth to compare against 98 w: width of image 99 h: height of image 100 e_msg_stem: customized string for error message 101 102 Returns: 103 error string if check fails 104 """ 105 thresh_ar = max(THRESH_AR_L, THRESH_AR_S + 106 max(w, h) * (THRESH_AR_L-THRESH_AR_S) / LARGE_SIZE_IMAGE) 107 ar = circle['w'] / circle['h'] 108 if not math.isclose(ar, ar_gt, abs_tol=thresh_ar): 109 e_msg = (f'{e_msg_stem} {w}x{h}: aspect_ratio {ar:.3f}, ' 110 f'thresh {thresh_ar:.3f}') 111 return e_msg 112 113 114def check_crop(circle, cc_gt, w, h, e_msg_stem, crop_thresh_factor): 115 """Check cropping. 116 117 if size >= LARGE_SIZE_IMAGE: use thresh_crop_l 118 elif size == 0 (extreme case): thresh_crop_s 119 elif 0 < size < LARGE_SIZE_IMAGE: scale between thresh_crop_s & thresh_crop_l 120 Also allow at least THRESH_MIN_PIXEL to prevent threshold being too tight 121 for very small circle. 122 123 Args: 124 circle: dict of circle values 125 cc_gt: circle center {'hori', 'vert'} ground truth (ref'd to img center) 126 w: width of image 127 h: height of image 128 e_msg_stem: text to customize error message 129 crop_thresh_factor: scaling factor for crop thresholds 130 131 Returns: 132 error string if check fails 133 """ 134 thresh_crop_l = THRESH_CROP_L * crop_thresh_factor 135 thresh_crop_s = THRESH_CROP_S * crop_thresh_factor 136 thresh_crop_hori = max( 137 [thresh_crop_l, 138 thresh_crop_s + w * (thresh_crop_l - thresh_crop_s) / LARGE_SIZE_IMAGE, 139 THRESH_MIN_PIXEL / circle['w']]) 140 thresh_crop_vert = max( 141 [thresh_crop_l, 142 thresh_crop_s + h * (thresh_crop_l - thresh_crop_s) / LARGE_SIZE_IMAGE, 143 THRESH_MIN_PIXEL / circle['h']]) 144 145 if (not math.isclose(circle['x_offset'], cc_gt['hori'], 146 abs_tol=thresh_crop_hori) or 147 not math.isclose(circle['y_offset'], cc_gt['vert'], 148 abs_tol=thresh_crop_vert)): 149 valid_x_range = (cc_gt['hori'] - thresh_crop_hori, 150 cc_gt['hori'] + thresh_crop_hori) 151 valid_y_range = (cc_gt['vert'] - thresh_crop_vert, 152 cc_gt['vert'] + thresh_crop_vert) 153 e_msg = (f'{e_msg_stem} {w}x{h} ' 154 f"offset X {circle['x_offset']:.3f}, Y {circle['y_offset']:.3f}, " 155 f'valid X range: {valid_x_range[0]:.3f} ~ {valid_x_range[1]:.3f}, ' 156 f'valid Y range: {valid_y_range[0]:.3f} ~ {valid_y_range[1]:.3f}') 157 return e_msg 158 159 160def calc_expected_circle_image_ratio(ref_fov, img_w, img_h): 161 """Determine the circle image area ratio in percentage for a given image size. 162 163 Cropping happens either horizontally or vertically. In both cases crop results 164 in the visble area reduced by a ratio r (r < 1) and the circle will in turn 165 occupy ref_pct/r (percent) on the target image size. 166 167 Args: 168 ref_fov: dict with {fmt, % coverage, w, h, circle_w, circle_h} 169 img_w: the image width 170 img_h: the image height 171 172 Returns: 173 chk_percent: the expected circle image area ratio in percentage 174 """ 175 ar_ref = ref_fov['w'] / ref_fov['h'] 176 ar_target = img_w / img_h 177 178 r = ar_ref / ar_target 179 if r < 1.0: 180 r = 1.0 / r 181 return ref_fov['percent'] * r 182 183 184def calc_circle_image_ratio(radius, img_w, img_h): 185 """Calculate the percent of area the input circle covers in input image. 186 187 Args: 188 radius: radius of circle 189 img_w: int width of image 190 img_h: int height of image 191 Returns: 192 fov_percent: float % of image covered by circle 193 """ 194 return 100 * math.pi * math.pow(radius, 2) / (img_w * img_h) 195 196 197def find_fov_reference(cam, req, props, raw_bool, ref_img_name_stem): 198 """Determine the circle coverage of the image in reference image. 199 200 Captures a full-frame RAW or JPEG and uses its aspect ratio and circle center 201 location as ground truth for the other jpeg or yuv images. 202 203 The intrinsics and distortion coefficients are meant for full-sized RAW, 204 so convert_capture_to_rgb_image returns a 2x downsampled version, so resizes 205 RGB back to full size. 206 207 If the device supports lens distortion correction, applies the coefficients on 208 the RAW image so it can be compared to YUV/JPEG outputs which are subject 209 to the same correction via ISP. 210 211 Finds circle size and location for reference values in calculations for other 212 formats. 213 214 Args: 215 cam: camera object 216 req: camera request 217 props: camera properties 218 raw_bool: True if RAW available 219 ref_img_name_stem: test _NAME + location to save data 220 221 Returns: 222 ref_fov: dict with {fmt, % coverage, w, h, circle_w, circle_h} 223 cc_ct_gt: circle center position relative to the center of image. 224 aspect_ratio_gt: aspect ratio of the detected circle in float. 225 """ 226 logging.debug('Creating references for fov_coverage') 227 if raw_bool: 228 logging.debug('Using RAW for reference') 229 fmt_type = 'RAW' 230 out_surface = {'format': 'raw'} 231 cap = cam.do_capture(req, out_surface) 232 logging.debug('Captured RAW %dx%d', cap['width'], cap['height']) 233 img = image_processing_utils.convert_capture_to_rgb_image( 234 cap, props=props) 235 # Resize back up to full scale. 236 img = cv2.resize(img, (0, 0), fx=2.0, fy=2.0) 237 238 fd = float(cap['metadata']['android.lens.focalLength']) 239 k = camera_properties_utils.get_intrinsic_calibration( 240 props, cap['metadata'], True, fd 241 ) 242 if (camera_properties_utils.distortion_correction(props) and 243 isinstance(k, np.ndarray)): 244 logging.debug('Applying intrinsic calibration and distortion params') 245 opencv_dist = camera_properties_utils.get_distortion_matrix(props) 246 k_new = cv2.getOptimalNewCameraMatrix( 247 k, opencv_dist, (img.shape[1], img.shape[0]), 0)[0] 248 scale = max(k_new[0][0] / k[0][0], k_new[1][1] / k[1][1]) 249 if scale > 1: 250 k_new[0][0] = k[0][0] * scale 251 k_new[1][1] = k[1][1] * scale 252 img = cv2.undistort(img, k, opencv_dist, None, k_new) 253 else: 254 img = cv2.undistort(img, k, opencv_dist) 255 size = img.shape 256 257 else: 258 logging.debug('Using JPEG for reference') 259 fmt_type = 'JPEG' 260 ref_fov = {} 261 fmt = capture_request_utils.get_largest_format('jpeg', props) 262 cap = cam.do_capture(req, fmt) 263 logging.debug('Captured JPEG %dx%d', cap['width'], cap['height']) 264 img = image_processing_utils.convert_capture_to_rgb_image(cap, props) 265 size = (cap['height'], cap['width']) 266 267 # Get image size. 268 w = size[1] 269 h = size[0] 270 img_name = f'{ref_img_name_stem}_{fmt_type}_w{w}_h{h}.png' 271 image_processing_utils.write_image(img, img_name, True) 272 273 # Find circle. 274 img *= 255 # cv2 needs images between [0,255]. 275 circle = opencv_processing_utils.find_circle( 276 img, img_name, CIRCLE_MIN_AREA, CIRCLE_COLOR) 277 opencv_processing_utils.append_circle_center_to_img(circle, img, img_name) 278 279 # Determine final return values. 280 if fmt_type == 'RAW': 281 aspect_ratio_gt = circle['w'] / circle['h'] 282 else: 283 aspect_ratio_gt = 1.0 284 cc_ct_gt = {'hori': circle['x_offset'], 'vert': circle['y_offset']} 285 fov_percent = calc_circle_image_ratio(circle['r'], w, h) 286 ref_fov = {} 287 ref_fov['fmt'] = fmt_type 288 ref_fov['percent'] = fov_percent 289 ref_fov['w'] = w 290 ref_fov['h'] = h 291 ref_fov['circle_w'] = circle['w'] 292 ref_fov['circle_h'] = circle['h'] 293 logging.debug('Using %s reference: %s', fmt_type, str(ref_fov)) 294 return ref_fov, cc_ct_gt, aspect_ratio_gt 295