1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Image Field-of-View utilities for aspect ratio, crop, and FoV tests.""" 15 16 17import logging 18import math 19 20import cv2 21import camera_properties_utils 22import capture_request_utils 23import image_processing_utils 24import opencv_processing_utils 25 26CIRCLE_COLOR = 0 # [0: black, 255: white] 27CIRCLE_MIN_AREA = 0.01 # 1% of image size 28FOV_PERCENT_RTOL = 0.15 # Relative tolerance on circle FoV % to expected. 29LARGE_SIZE_IMAGE = 2000 # Size of a large image (compared against max(w, h)) 30THRESH_AR_L = 0.02 # Aspect ratio test threshold of large images 31THRESH_AR_S = 0.075 # Aspect ratio test threshold of mini images 32THRESH_CROP_L = 0.02 # Crop test threshold of large images 33THRESH_CROP_S = 0.075 # Crop test threshold of mini images 34THRESH_MIN_PIXEL = 4 # Crop test allowed offset 35 36 37def check_fov(circle, ref_fov, w, h): 38 """Check the FoV for correct size.""" 39 fov_percent = calc_circle_image_ratio(circle['r'], w, h) 40 chk_percent = calc_expected_circle_image_ratio(ref_fov, w, h) 41 if not math.isclose(fov_percent, chk_percent, rel_tol=FOV_PERCENT_RTOL): 42 e_msg = (f'FoV %: {fov_percent:.2f}, Ref FoV %: {chk_percent:.2f}, ' 43 f'TOL={FOV_PERCENT_RTOL*100}%, img: {w}x{h}, ref: ' 44 f"{ref_fov['w']}x{ref_fov['h']}") 45 return e_msg 46 47 48def check_ar(circle, ar_gt, w, h, e_msg_stem): 49 """Check the aspect ratio of the circle. 50 51 size is the larger of w or h. 52 if size >= LARGE_SIZE_IMAGE: use THRESH_AR_L 53 elif size == 0 (extreme case): THRESH_AR_S 54 elif 0 < image size < LARGE_SIZE_IMAGE: scale between THRESH_AR_S & AR_L 55 56 Args: 57 circle: dict with circle parameters 58 ar_gt: aspect ratio ground truth to compare against 59 w: width of image 60 h: height of image 61 e_msg_stem: customized string for error message 62 63 Returns: 64 error string if check fails 65 """ 66 thresh_ar = max(THRESH_AR_L, THRESH_AR_S + 67 max(w, h) * (THRESH_AR_L-THRESH_AR_S) / LARGE_SIZE_IMAGE) 68 ar = circle['w'] / circle['h'] 69 if not math.isclose(ar, ar_gt, abs_tol=thresh_ar): 70 e_msg = (f'{e_msg_stem} {w}x{h}: aspect_ratio {ar:.3f}, ' 71 f'thresh {thresh_ar:.3f}') 72 return e_msg 73 74 75def check_crop(circle, cc_gt, w, h, e_msg_stem, crop_thresh_factor): 76 """Check cropping. 77 78 if size >= LARGE_SIZE_IMAGE: use thresh_crop_l 79 elif size == 0 (extreme case): thresh_crop_s 80 elif 0 < size < LARGE_SIZE_IMAGE: scale between thresh_crop_s & thresh_crop_l 81 Also allow at least THRESH_MIN_PIXEL to prevent threshold being too tight 82 for very small circle. 83 84 Args: 85 circle: dict of circle values 86 cc_gt: circle center {'hori', 'vert'} ground truth (ref'd to img center) 87 w: width of image 88 h: height of image 89 e_msg_stem: text to customize error message 90 crop_thresh_factor: scaling factor for crop thresholds 91 92 Returns: 93 error string if check fails 94 """ 95 thresh_crop_l = THRESH_CROP_L * crop_thresh_factor 96 thresh_crop_s = THRESH_CROP_S * crop_thresh_factor 97 thresh_crop_hori = max( 98 [thresh_crop_l, 99 thresh_crop_s + w * (thresh_crop_l - thresh_crop_s) / LARGE_SIZE_IMAGE, 100 THRESH_MIN_PIXEL / circle['w']]) 101 thresh_crop_vert = max( 102 [thresh_crop_l, 103 thresh_crop_s + h * (thresh_crop_l - thresh_crop_s) / LARGE_SIZE_IMAGE, 104 THRESH_MIN_PIXEL / circle['h']]) 105 106 if (not math.isclose(circle['x_offset'], cc_gt['hori'], 107 abs_tol=thresh_crop_hori) or 108 not math.isclose(circle['y_offset'], cc_gt['vert'], 109 abs_tol=thresh_crop_vert)): 110 valid_x_range = (cc_gt['hori'] - thresh_crop_hori, 111 cc_gt['hori'] + thresh_crop_hori) 112 valid_y_range = (cc_gt['vert'] - thresh_crop_vert, 113 cc_gt['vert'] + thresh_crop_vert) 114 e_msg = (f'{e_msg_stem} {w}x{h} ' 115 f"offset X {circle['x_offset']:.3f}, Y {circle['y_offset']:.3f}, " 116 f'valid X range: {valid_x_range[0]:.3f} ~ {valid_x_range[1]:.3f}, ' 117 f'valid Y range: {valid_y_range[0]:.3f} ~ {valid_y_range[1]:.3f}') 118 return e_msg 119 120 121def calc_expected_circle_image_ratio(ref_fov, img_w, img_h): 122 """Determine the circle image area ratio in percentage for a given image size. 123 124 Cropping happens either horizontally or vertically. In both cases crop results 125 in the visble area reduced by a ratio r (r < 1) and the circle will in turn 126 occupy ref_pct/r (percent) on the target image size. 127 128 Args: 129 ref_fov: dict with {fmt, % coverage, w, h, circle_w, circle_h} 130 img_w: the image width 131 img_h: the image height 132 133 Returns: 134 chk_percent: the expected circle image area ratio in percentage 135 """ 136 ar_ref = ref_fov['w'] / ref_fov['h'] 137 ar_target = img_w / img_h 138 139 r = ar_ref / ar_target 140 if r < 1.0: 141 r = 1.0 / r 142 return ref_fov['percent'] * r 143 144 145def calc_circle_image_ratio(radius, img_w, img_h): 146 """Calculate the percent of area the input circle covers in input image. 147 148 Args: 149 radius: radius of circle 150 img_w: int width of image 151 img_h: int height of image 152 Returns: 153 fov_percent: float % of image covered by circle 154 """ 155 return 100 * math.pi * math.pow(radius, 2) / (img_w * img_h) 156 157 158def find_fov_reference(cam, req, props, raw_bool, ref_img_name_stem): 159 """Determine the circle coverage of the image in reference image. 160 161 Captures a full-frame RAW or JPEG and uses its aspect ratio and circle center 162 location as ground truth for the other jpeg or yuv images. 163 164 The intrinsics and distortion coefficients are meant for full-sized RAW, 165 so convert_capture_to_rgb_image returns a 2x downsampled version, so resizes 166 RGB back to full size. 167 168 If the device supports lens distortion correction, applies the coefficients on 169 the RAW image so it can be compared to YUV/JPEG outputs which are subject 170 to the same correction via ISP. 171 172 Finds circle size and location for reference values in calculations for other 173 formats. 174 175 Args: 176 cam: camera object 177 req: camera request 178 props: camera properties 179 raw_bool: True if RAW available 180 ref_img_name_stem: test _NAME + location to save data 181 182 Returns: 183 ref_fov: dict with {fmt, % coverage, w, h, circle_w, circle_h} 184 cc_ct_gt: circle center position relative to the center of image. 185 aspect_ratio_gt: aspect ratio of the detected circle in float. 186 """ 187 logging.debug('Creating references for fov_coverage') 188 if raw_bool: 189 logging.debug('Using RAW for reference') 190 fmt_type = 'RAW' 191 out_surface = {'format': 'raw'} 192 cap = cam.do_capture(req, out_surface) 193 logging.debug('Captured RAW %dx%d', cap['width'], cap['height']) 194 img = image_processing_utils.convert_capture_to_rgb_image( 195 cap, props=props) 196 # Resize back up to full scale. 197 img = cv2.resize(img, (0, 0), fx=2.0, fy=2.0) 198 199 if (camera_properties_utils.distortion_correction(props) and 200 camera_properties_utils.intrinsic_calibration(props)): 201 logging.debug('Applying intrinsic calibration and distortion params') 202 fd = float(cap['metadata']['android.lens.focalLength']) 203 k = camera_properties_utils.get_intrinsic_calibration(props, True, fd) 204 opencv_dist = camera_properties_utils.get_distortion_matrix(props) 205 k_new = cv2.getOptimalNewCameraMatrix( 206 k, opencv_dist, (img.shape[1], img.shape[0]), 0)[0] 207 scale = max(k_new[0][0] / k[0][0], k_new[1][1] / k[1][1]) 208 if scale > 1: 209 k_new[0][0] = k[0][0] * scale 210 k_new[1][1] = k[1][1] * scale 211 img = cv2.undistort(img, k, opencv_dist, None, k_new) 212 else: 213 img = cv2.undistort(img, k, opencv_dist) 214 size = img.shape 215 216 else: 217 logging.debug('Using JPEG for reference') 218 fmt_type = 'JPEG' 219 ref_fov = {} 220 fmt = capture_request_utils.get_largest_jpeg_format(props) 221 cap = cam.do_capture(req, fmt) 222 logging.debug('Captured JPEG %dx%d', cap['width'], cap['height']) 223 img = image_processing_utils.convert_capture_to_rgb_image(cap, props) 224 size = (cap['height'], cap['width']) 225 226 # Get image size. 227 w = size[1] 228 h = size[0] 229 img_name = f'{ref_img_name_stem}_{fmt_type}_w{w}_h{h}.png' 230 image_processing_utils.write_image(img, img_name, True) 231 232 # Find circle. 233 img *= 255 # cv2 needs images between [0,255]. 234 circle = opencv_processing_utils.find_circle( 235 img, img_name, CIRCLE_MIN_AREA, CIRCLE_COLOR) 236 opencv_processing_utils.append_circle_center_to_img(circle, img, img_name) 237 238 # Determine final return values. 239 if fmt_type == 'RAW': 240 aspect_ratio_gt = circle['w'] / circle['h'] 241 else: 242 aspect_ratio_gt = 1.0 243 cc_ct_gt = {'hori': circle['x_offset'], 'vert': circle['y_offset']} 244 fov_percent = calc_circle_image_ratio(circle['r'], w, h) 245 ref_fov = {} 246 ref_fov['fmt'] = fmt_type 247 ref_fov['percent'] = fov_percent 248 ref_fov['w'] = w 249 ref_fov['h'] = h 250 ref_fov['circle_w'] = circle['w'] 251 ref_fov['circle_h'] = circle['h'] 252 logging.debug('Using %s reference: %s', fmt_type, str(ref_fov)) 253 return ref_fov, cc_ct_gt, aspect_ratio_gt 254