1# Copyright 2015 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Validate aspect ratio, crop and FoV vs format.""" 15 16 17import logging 18import math 19import os.path 20from mobly import test_runner 21 22import its_base_test 23import camera_properties_utils 24import capture_request_utils 25import image_fov_utils 26import image_processing_utils 27import its_session_utils 28import opencv_processing_utils 29 30_ANDROID11_API_LEVEL = 30 31_NAME = os.path.splitext(os.path.basename(__file__))[0] 32_SIZE_PREVIEW = (1920, 1080) 33_SIZE_PREVIEW_4x3 = (1440, 1080) 34_SIZE_VGA = (640, 480) 35_SIZES_COMMON = ( 36 (1920, 1080), 37 (1440, 1080), 38 (1280, 720), 39 (960, 720), 40 (640, 480), 41) 42 43 44# Before API level 30, only resolutions with the following listed aspect ratio 45# are checked. Device launched after API level 30 will need to pass the test 46# for all advertised resolutions. Device launched before API level 30 just 47# needs to pass the test for all resolutions within these aspect ratios. 48_AR_CHECKED_PRE_API_30 = ('4:3', '16:9', '18:9') 49_AR_DIFF_ATOL = 0.01 50# If RAW reference capture aspect ratio is ~4:3 or ~16:9, use JPEG, else RAW 51_AR_FOR_JPEG_REFERENCE = (4/3, 16/9) 52 53 54def _check_skip_conditions(first_api_level, props): 55 """Check the skip conditions based on first API level.""" 56 if first_api_level < _ANDROID11_API_LEVEL: # Original constraint. 57 camera_properties_utils.skip_unless(camera_properties_utils.read_3a(props)) 58 else: # Loosen from read_3a to enable LIMITED coverage. 59 camera_properties_utils.skip_unless( 60 camera_properties_utils.ae_lock(props) and 61 camera_properties_utils.awb_lock(props)) 62 63 64def _check_basic_correctness(cap, fmt_iter, w_iter, h_iter): 65 """Check the capture for basic correctness.""" 66 if cap['format'] != fmt_iter: 67 raise AssertionError 68 if cap['width'] != w_iter: 69 raise AssertionError 70 if cap['height'] != h_iter: 71 raise AssertionError 72 73 74def _create_format_list(): 75 """Create format list for multiple capture objects. 76 77 Do multi-capture of 'iter' and 'cmpr'. Iterate through all the available 78 sizes of 'iter', and only use the size specified for 'cmpr'. 79 The 'cmpr' capture is only used so that we have multiple capture target 80 instead of just one, which should help catching more potential issues. 81 The test doesn't look into the output of 'cmpr' images at all. 82 The 'iter_max' or 'cmpr_size' key defines the maximal size being iterated 83 or selected for the 'iter' and 'cmpr' stream accordingly. None means no 84 upper bound is specified. 85 86 Args: 87 None 88 89 Returns: 90 format_list 91 """ 92 format_list = [] 93 format_list.append({'iter': 'jpeg_r', 'iter_max': None, 94 'cmpr': 'priv', 'cmpr_size': _SIZE_PREVIEW}) 95 format_list.append({'iter': 'yuv', 'iter_max': None, 96 'cmpr': 'yuv', 'cmpr_size': _SIZE_PREVIEW}) 97 format_list.append({'iter': 'yuv', 'iter_max': _SIZE_PREVIEW, 98 'cmpr': 'jpeg', 'cmpr_size': None}) 99 format_list.append({'iter': 'yuv', 'iter_max': _SIZE_PREVIEW, 100 'cmpr': 'raw', 'cmpr_size': None}) 101 format_list.append({'iter': 'jpeg', 'iter_max': None, 102 'cmpr': 'raw', 'cmpr_size': None}) 103 format_list.append({'iter': 'jpeg', 'iter_max': None, 104 'cmpr': 'yuv', 'cmpr_size': _SIZE_PREVIEW}) 105 format_list.append({'iter': 'yuv', 'iter_max': None, 106 'cmpr': 'priv', 'cmpr_size': _SIZE_PREVIEW}) 107 format_list.append({'iter': 'yuv', 'iter_max': None, 108 'cmpr': 'priv', 'cmpr_size': _SIZE_PREVIEW_4x3}) 109 format_list.append({'iter': 'yuv', 'iter_max': _SIZE_VGA, 110 'cmpr': 'priv', 'cmpr_size': _SIZE_PREVIEW, 111 'third': 'yuv', 'third_size': _SIZE_PREVIEW}) 112 return format_list 113 114 115def _print_failed_test_results(failed_ar, failed_fov, failed_crop, 116 first_api_level, level_3): 117 """Print failed test results.""" 118 if failed_ar: 119 logging.error('Aspect ratio test summary') 120 logging.error('Images failed in the aspect ratio test:') 121 logging.error('Aspect ratio value: width / height') 122 for fa in failed_ar: 123 logging.error('%s', fa) 124 125 if failed_fov: 126 logging.error('FoV test summary') 127 logging.error('Images failed in the FoV test:') 128 for fov in failed_fov: 129 logging.error('%s', str(fov)) 130 131 if failed_crop: 132 logging.error('Crop test summary') 133 logging.error('Images failed in the crop test:') 134 logging.error('Circle center (H x V) relative to the image center.') 135 for fc in failed_crop: 136 logging.error('%s', fc) 137 if failed_ar: 138 raise RuntimeError 139 if failed_fov: 140 raise RuntimeError 141 if first_api_level > _ANDROID11_API_LEVEL: 142 if failed_crop: # failed_crop = [] if run_crop_test = False. 143 raise RuntimeError 144 else: 145 if failed_crop and level_3: 146 raise RuntimeError 147 148 149def _is_checked_aspect_ratio(first_api_level, w, h): 150 """Determine if format aspect ratio is a checked on based of first_API.""" 151 if first_api_level >= _ANDROID11_API_LEVEL: 152 return True 153 154 for ar_check in _AR_CHECKED_PRE_API_30: 155 match_ar_list = [float(x) for x in ar_check.split(':')] 156 match_ar = match_ar_list[0] / match_ar_list[1] 157 if math.isclose(w / h, match_ar, abs_tol=_AR_DIFF_ATOL): 158 return True 159 160 return False 161 162 163class AspectRatioAndCropTest(its_base_test.ItsBaseTest): 164 """Test aspect ratio/field of view/cropping for each tested fmt combinations. 165 166 This test checks for: 167 1. Aspect ratio: images are not stretched 168 2. Crop: center of images is not shifted 169 3. FOV: images cropped to keep maximum possible FOV with only 1 dimension 170 (horizontal or veritical) cropped. 171 172 Aspect ratio and FOV test runs on level3, full and limited devices. 173 Crop test only runs on level3 and full devices. 174 175 The test chart is a black circle inside a black square. When raw capture is 176 available, set the height vs. width ratio of the circle in the full-frame 177 raw as ground truth. In an ideal setup such ratio should be very close to 178 1.0, but here we just use the value derived from full resolution RAW as 179 ground truth to account for the possibility that the chart is not well 180 positioned to be precisely parallel to image sensor plane. 181 The test then compares the ground truth ratio with the same ratio measured 182 on images captured using different stream combinations of varying formats 183 ('jpeg' and 'yuv') and resolutions. 184 If raw capture is unavailable, a full resolution JPEG image is used to setup 185 ground truth. In this case, the ground truth aspect ratio is defined as 1.0 186 and it is the tester's responsibility to make sure the test chart is 187 properly positioned so the detected circles indeed have aspect ratio close 188 to 1.0 assuming no bugs causing image stretched. 189 190 The aspect ratio test checks the aspect ratio of the detected circle and 191 it will fail if the aspect ratio differs too much from the ground truth 192 aspect ratio mentioned above. 193 194 The FOV test examines the ratio between the detected circle area and the 195 image size. When the aspect ratio of the test image is the same as the 196 ground truth image, the ratio should be very close to the ground truth 197 value. When the aspect ratio is different, the difference is factored in 198 per the expectation of the Camera2 API specification, which mandates the 199 FOV reduction from full sensor area must only occur in one dimension: 200 horizontally or vertically, and never both. For example, let's say a sensor 201 has a 16:10 full sensor FOV. For all 16:10 output images there should be no 202 FOV reduction on them. For 16:9 output images the FOV should be vertically 203 cropped by 9/10. For 4:3 output images the FOV should be cropped 204 horizontally instead and the ratio (r) can be calculated as follows: 205 (16 * r) / 10 = 4 / 3 => r = 40 / 48 = 0.8333 206 Say the circle is covering x percent of the 16:10 sensor on the full 16:10 207 FOV, and assume the circle in the center will never be cut in any output 208 sizes (this can be achieved by picking the right size and position of the 209 test circle), the from above cropping expectation we can derive on a 16:9 210 output image the circle will cover (x / 0.9) percent of the 16:9 image; on 211 a 4:3 output image the circle will cover (x / 0.8333) percent of the 4:3 212 image. 213 214 The crop test checks that the center of any output image remains aligned 215 with center of sensor's active area, no matter what kind of cropping or 216 scaling is applied. The test verifies that by checking the relative vector 217 from the image center to the center of detected circle remains unchanged. 218 The relative part is normalized by the detected circle size to account for 219 scaling effect. 220 """ 221 222 def test_aspect_ratio_and_crop(self): 223 logging.debug('Starting %s', _NAME) 224 failed_ar = [] # Streams failed the aspect ratio test. 225 failed_crop = [] # Streams failed the crop test. 226 failed_fov = [] # Streams that fail FoV test. 227 format_list = _create_format_list() 228 229 with its_session_utils.ItsSession( 230 device_id=self.dut.serial, 231 camera_id=self.camera_id, 232 hidden_physical_id=self.hidden_physical_id) as cam: 233 props = cam.get_camera_properties() 234 fls_logical = props['android.lens.info.availableFocalLengths'] 235 logging.debug('logical available focal lengths: %s', str(fls_logical)) 236 props = cam.override_with_hidden_physical_camera_props(props) 237 fls_physical = props['android.lens.info.availableFocalLengths'] 238 logging.debug('physical available focal lengths: %s', str(fls_physical)) 239 logging.debug('minimum focus distance (diopters): %.2f', 240 props['android.lens.info.minimumFocusDistance']) 241 name_with_log_path = os.path.join(self.log_path, _NAME) 242 if self.hidden_physical_id: 243 logging.debug('Testing camera: %s.%s', 244 self.camera_id, self.hidden_physical_id) 245 246 # Check SKIP conditions. 247 first_api_level = its_session_utils.get_first_api_level(self.dut.serial) 248 _check_skip_conditions(first_api_level, props) 249 250 # Load chart for scene. 251 its_session_utils.load_scene( 252 cam, props, self.scene, self.tablet, self.chart_distance) 253 254 # Determine camera capabilities. 255 full_or_better = camera_properties_utils.full_or_better(props) 256 level3 = camera_properties_utils.level3(props) 257 raw_avlb = camera_properties_utils.raw16(props) 258 259 # Converge 3A. 260 if camera_properties_utils.manual_sensor(props): 261 logging.debug('Manual sensor, using manual capture request') 262 s, e, _, _, f_d = cam.do_3a(get_results=True) 263 req = capture_request_utils.manual_capture_request( 264 s, e, f_distance=f_d) 265 else: 266 logging.debug('Using auto capture request') 267 cam.do_3a() 268 req = capture_request_utils.auto_capture_request() 269 270 # For main camera: if RAW available, use it as ground truth, else JPEG 271 # For physical sub-camera: if RAW available, only use if not 4:3 or 16:9 272 use_raw_fov = False 273 if raw_avlb: 274 pixel_array_w = props['android.sensor.info.pixelArraySize']['width'] 275 pixel_array_h = props['android.sensor.info.pixelArraySize']['height'] 276 logging.debug('Pixel array size: %dx%d', pixel_array_w, pixel_array_h) 277 raw_aspect_ratio = pixel_array_w / pixel_array_h 278 use_raw_fov = ( 279 fls_physical == fls_logical or not 280 any(math.isclose(raw_aspect_ratio, jpeg_ar, abs_tol=_AR_DIFF_ATOL) 281 for jpeg_ar in _AR_FOR_JPEG_REFERENCE) 282 ) 283 284 ref_fov, cc_ct_gt, aspect_ratio_gt = ( 285 image_fov_utils.find_fov_reference( 286 cam, req, props, use_raw_fov, name_with_log_path)) 287 288 run_crop_test = full_or_better and raw_avlb 289 if run_crop_test: 290 # Normalize the circle size to 1/4 of the image size, so that 291 # circle size won't affect the crop test result 292 crop_thresh_factor = ((min(ref_fov['w'], ref_fov['h']) / 4.0) / 293 max(ref_fov['circle_w'], ref_fov['circle_h'])) 294 else: 295 logging.debug('Crop test skipped') 296 297 # Take pictures of each settings with all the image sizes available. 298 for fmt in format_list: 299 fmt_iter = fmt['iter'] 300 fmt_cmpr = fmt['cmpr'] 301 # Get the size of 'cmpr'. 302 sizes = capture_request_utils.get_available_output_sizes( 303 fmt_cmpr, props, fmt['cmpr_size']) 304 if not sizes: # Device might not support RAW. 305 continue 306 w_cmpr, h_cmpr = sizes[0][0], sizes[0][1] 307 # Get the size of third stream if defined. 308 if 'third' in fmt.keys(): 309 sizes_third = capture_request_utils.get_available_output_sizes( 310 fmt_cmpr, props, fmt['third_size']) 311 test_sizes = capture_request_utils.get_available_output_sizes( 312 fmt_iter, props, fmt['iter_max']) 313 if fmt_cmpr == its_session_utils.PRIVATE_FORMAT: 314 test_sizes = [size for size in test_sizes if size in _SIZES_COMMON] 315 for size_iter in test_sizes: 316 w_iter, h_iter = size_iter[0], size_iter[1] 317 # Skip same format/size combination: ITS doesn't handle that properly. 318 if w_iter*h_iter == w_cmpr*h_cmpr and fmt_iter == fmt_cmpr: 319 continue 320 out_surface = [{'width': w_iter, 'height': h_iter, 321 'format': fmt_iter}] 322 out_surface.append({'width': w_cmpr, 'height': h_cmpr, 323 'format': fmt_cmpr}) 324 if 'third' in fmt.keys(): 325 out_surface.append({'width': sizes_third[0][0], 326 'height': sizes_third[0][1], 327 'format': fmt['third']}) 328 if cam.is_stream_combination_supported(out_surface): 329 cap = cam.do_capture(req, out_surface)[0] 330 _check_basic_correctness(cap, fmt_iter, w_iter, h_iter) 331 logging.debug('Captured %s with %s %dx%d. Compared size: %dx%d', 332 fmt_iter, fmt_cmpr, w_iter, h_iter, w_cmpr, h_cmpr) 333 img = image_processing_utils.convert_capture_to_rgb_image(cap) 334 img *= 255 # cv2 uses [0, 255]. 335 img_name = f'{name_with_log_path}_{fmt_iter}_with_{fmt_cmpr}_w{w_iter}_h{h_iter}.png' 336 circle = opencv_processing_utils.find_circle( 337 img, img_name, image_fov_utils.CIRCLE_MIN_AREA, 338 image_fov_utils.CIRCLE_COLOR) 339 opencv_processing_utils.append_circle_center_to_img( 340 circle, img, img_name, save_img=False) # imgs saved on FAILs 341 342 # Check pass/fail for fov coverage for all fmts in AR_CHECKED 343 img /= 255 # image_processing_utils uses [0, 1]. 344 if _is_checked_aspect_ratio(first_api_level, w_iter, h_iter): 345 fov_chk_msg = image_fov_utils.check_fov( 346 circle, ref_fov, w_iter, h_iter) 347 if fov_chk_msg: 348 failed_fov.append(fov_chk_msg) 349 image_processing_utils.write_image(img, img_name, True) 350 351 # Check pass/fail for aspect ratio. 352 ar_chk_msg = image_fov_utils.check_ar( 353 circle, aspect_ratio_gt, w_iter, h_iter, 354 f'{fmt_iter} with {fmt_cmpr}') 355 if ar_chk_msg: 356 failed_ar.append(ar_chk_msg) 357 image_processing_utils.write_image(img, img_name, True) 358 359 # Check pass/fail for crop. 360 if run_crop_test: 361 crop_chk_msg = image_fov_utils.check_crop( 362 circle, cc_ct_gt, w_iter, h_iter, 363 f'{fmt_iter} with {fmt_cmpr}', crop_thresh_factor) 364 if crop_chk_msg: 365 failed_crop.append(crop_chk_msg) 366 image_processing_utils.write_image(img, img_name, True) 367 else: 368 continue 369 370 # Print any failed test results. 371 _print_failed_test_results(failed_ar, failed_fov, failed_crop, 372 first_api_level, level3) 373 374if __name__ == '__main__': 375 test_runner.main() 376