1# Copyright 2015 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Validate aspect ratio, crop and FoV vs format.""" 15 16 17import logging 18import math 19import os.path 20from mobly import test_runner 21 22import its_base_test 23import camera_properties_utils 24import capture_request_utils 25import image_fov_utils 26import image_processing_utils 27import its_session_utils 28import opencv_processing_utils 29 30_ANDROID11_API_LEVEL = 30 31_NAME = os.path.splitext(os.path.basename(__file__))[0] 32_PREVIEW_SIZE = (1920, 1080) 33 34# Before API level 30, only resolutions with the following listed aspect ratio 35# are checked. Device launched after API level 30 will need to pass the test 36# for all advertised resolutions. Device launched before API level 30 just 37# needs to pass the test for all resolutions within these aspect ratios. 38_AR_CHECKED_PRE_API_30 = ('4:3', '16:9', '18:9') 39_AR_DIFF_ATOL = 0.01 40# If RAW reference capture aspect ratio is ~4:3 or ~16:9, use JPEG, else RAW 41_AR_FOR_JPEG_REFERENCE = (4/3, 16/9) 42 43 44def _check_skip_conditions(first_api_level, props): 45 """Check the skip conditions based on first API level.""" 46 if first_api_level < _ANDROID11_API_LEVEL: # Original constraint. 47 camera_properties_utils.skip_unless(camera_properties_utils.read_3a(props)) 48 else: # Loosen from read_3a to enable LIMITED coverage. 49 camera_properties_utils.skip_unless( 50 camera_properties_utils.ae_lock(props) and 51 camera_properties_utils.awb_lock(props)) 52 53 54def _check_basic_correctness(cap, fmt_iter, w_iter, h_iter): 55 """Check the capture for basic correctness.""" 56 if cap['format'] != fmt_iter: 57 raise AssertionError 58 if cap['width'] != w_iter: 59 raise AssertionError 60 if cap['height'] != h_iter: 61 raise AssertionError 62 63 64def _create_format_list(): 65 """Create format list for multiple capture objects. 66 67 Do multi-capture of 'iter' and 'cmpr'. Iterate through all the available 68 sizes of 'iter', and only use the size specified for 'cmpr'. 69 The 'cmpr' capture is only used so that we have multiple capture target 70 instead of just one, which should help catching more potential issues. 71 The test doesn't look into the output of 'cmpr' images at all. 72 The 'iter_max' or 'cmpr_size' key defines the maximal size being iterated 73 or selected for the 'iter' and 'cmpr' stream accordingly. None means no 74 upper bound is specified. 75 76 Args: 77 None 78 79 Returns: 80 format_list 81 """ 82 format_list = [] 83 format_list.append({'iter': 'jpeg_r', 'iter_max': None, 84 'cmpr': 'priv', 'cmpr_size': _PREVIEW_SIZE}) 85 format_list.append({'iter': 'yuv', 'iter_max': None, 86 'cmpr': 'yuv', 'cmpr_size': _PREVIEW_SIZE}) 87 format_list.append({'iter': 'yuv', 'iter_max': _PREVIEW_SIZE, 88 'cmpr': 'jpeg', 'cmpr_size': None}) 89 format_list.append({'iter': 'yuv', 'iter_max': _PREVIEW_SIZE, 90 'cmpr': 'raw', 'cmpr_size': None}) 91 format_list.append({'iter': 'jpeg', 'iter_max': None, 92 'cmpr': 'raw', 'cmpr_size': None}) 93 format_list.append({'iter': 'jpeg', 'iter_max': None, 94 'cmpr': 'yuv', 'cmpr_size': _PREVIEW_SIZE}) 95 return format_list 96 97 98def _print_failed_test_results(failed_ar, failed_fov, failed_crop, 99 first_api_level, level_3): 100 """Print failed test results.""" 101 if failed_ar: 102 logging.error('Aspect ratio test summary') 103 logging.error('Images failed in the aspect ratio test:') 104 logging.error('Aspect ratio value: width / height') 105 for fa in failed_ar: 106 logging.error('%s', fa) 107 108 if failed_fov: 109 logging.error('FoV test summary') 110 logging.error('Images failed in the FoV test:') 111 for fov in failed_fov: 112 logging.error('%s', str(fov)) 113 114 if failed_crop: 115 logging.error('Crop test summary') 116 logging.error('Images failed in the crop test:') 117 logging.error('Circle center (H x V) relative to the image center.') 118 for fc in failed_crop: 119 logging.error('%s', fc) 120 if failed_ar: 121 raise RuntimeError 122 if failed_fov: 123 raise RuntimeError 124 if first_api_level > _ANDROID11_API_LEVEL: 125 if failed_crop: # failed_crop = [] if run_crop_test = False. 126 raise RuntimeError 127 else: 128 if failed_crop and level_3: 129 raise RuntimeError 130 131 132def _is_checked_aspect_ratio(first_api_level, w, h): 133 """Determine if format aspect ratio is a checked on based of first_API.""" 134 if first_api_level >= _ANDROID11_API_LEVEL: 135 return True 136 137 for ar_check in _AR_CHECKED_PRE_API_30: 138 match_ar_list = [float(x) for x in ar_check.split(':')] 139 match_ar = match_ar_list[0] / match_ar_list[1] 140 if math.isclose(w / h, match_ar, abs_tol=_AR_DIFF_ATOL): 141 return True 142 143 return False 144 145 146class AspectRatioAndCropTest(its_base_test.ItsBaseTest): 147 """Test aspect ratio/field of view/cropping for each tested fmt combinations. 148 149 This test checks for: 150 1. Aspect ratio: images are not stretched 151 2. Crop: center of images is not shifted 152 3. FOV: images cropped to keep maximum possible FOV with only 1 dimension 153 (horizontal or veritical) cropped. 154 155 Aspect ratio and FOV test runs on level3, full and limited devices. 156 Crop test only runs on level3 and full devices. 157 158 The test chart is a black circle inside a black square. When raw capture is 159 available, set the height vs. width ratio of the circle in the full-frame 160 raw as ground truth. In an ideal setup such ratio should be very close to 161 1.0, but here we just use the value derived from full resolution RAW as 162 ground truth to account for the possibility that the chart is not well 163 positioned to be precisely parallel to image sensor plane. 164 The test then compares the ground truth ratio with the same ratio measured 165 on images captured using different stream combinations of varying formats 166 ('jpeg' and 'yuv') and resolutions. 167 If raw capture is unavailable, a full resolution JPEG image is used to setup 168 ground truth. In this case, the ground truth aspect ratio is defined as 1.0 169 and it is the tester's responsibility to make sure the test chart is 170 properly positioned so the detected circles indeed have aspect ratio close 171 to 1.0 assuming no bugs causing image stretched. 172 173 The aspect ratio test checks the aspect ratio of the detected circle and 174 it will fail if the aspect ratio differs too much from the ground truth 175 aspect ratio mentioned above. 176 177 The FOV test examines the ratio between the detected circle area and the 178 image size. When the aspect ratio of the test image is the same as the 179 ground truth image, the ratio should be very close to the ground truth 180 value. When the aspect ratio is different, the difference is factored in 181 per the expectation of the Camera2 API specification, which mandates the 182 FOV reduction from full sensor area must only occur in one dimension: 183 horizontally or vertically, and never both. For example, let's say a sensor 184 has a 16:10 full sensor FOV. For all 16:10 output images there should be no 185 FOV reduction on them. For 16:9 output images the FOV should be vertically 186 cropped by 9/10. For 4:3 output images the FOV should be cropped 187 horizontally instead and the ratio (r) can be calculated as follows: 188 (16 * r) / 10 = 4 / 3 => r = 40 / 48 = 0.8333 189 Say the circle is covering x percent of the 16:10 sensor on the full 16:10 190 FOV, and assume the circle in the center will never be cut in any output 191 sizes (this can be achieved by picking the right size and position of the 192 test circle), the from above cropping expectation we can derive on a 16:9 193 output image the circle will cover (x / 0.9) percent of the 16:9 image; on 194 a 4:3 output image the circle will cover (x / 0.8333) percent of the 4:3 195 image. 196 197 The crop test checks that the center of any output image remains aligned 198 with center of sensor's active area, no matter what kind of cropping or 199 scaling is applied. The test verifies that by checking the relative vector 200 from the image center to the center of detected circle remains unchanged. 201 The relative part is normalized by the detected circle size to account for 202 scaling effect. 203 """ 204 205 def test_aspect_ratio_and_crop(self): 206 logging.debug('Starting %s', _NAME) 207 failed_ar = [] # Streams failed the aspect ratio test. 208 failed_crop = [] # Streams failed the crop test. 209 failed_fov = [] # Streams that fail FoV test. 210 format_list = _create_format_list() 211 212 with its_session_utils.ItsSession( 213 device_id=self.dut.serial, 214 camera_id=self.camera_id, 215 hidden_physical_id=self.hidden_physical_id) as cam: 216 props = cam.get_camera_properties() 217 fls_logical = props['android.lens.info.availableFocalLengths'] 218 logging.debug('logical available focal lengths: %s', str(fls_logical)) 219 props = cam.override_with_hidden_physical_camera_props(props) 220 fls_physical = props['android.lens.info.availableFocalLengths'] 221 logging.debug('physical available focal lengths: %s', str(fls_physical)) 222 name_with_log_path = os.path.join(self.log_path, _NAME) 223 if self.hidden_physical_id: 224 logging.debug('Testing camera: %s.%s', 225 self.camera_id, self.hidden_physical_id) 226 227 # Check SKIP conditions. 228 first_api_level = its_session_utils.get_first_api_level(self.dut.serial) 229 _check_skip_conditions(first_api_level, props) 230 231 # Load chart for scene. 232 its_session_utils.load_scene( 233 cam, props, self.scene, self.tablet, self.chart_distance) 234 235 # Determine camera capabilities. 236 full_or_better = camera_properties_utils.full_or_better(props) 237 level3 = camera_properties_utils.level3(props) 238 raw_avlb = camera_properties_utils.raw16(props) 239 debug = self.debug_mode 240 241 # Converge 3A. 242 if camera_properties_utils.manual_sensor(props): 243 logging.debug('Manual sensor, using manual capture request') 244 s, e, _, _, f_d = cam.do_3a(get_results=True) 245 req = capture_request_utils.manual_capture_request( 246 s, e, f_distance=f_d) 247 else: 248 logging.debug('Using auto capture request') 249 cam.do_3a() 250 req = capture_request_utils.auto_capture_request() 251 252 # For main camera: if RAW available, use it as ground truth, else JPEG 253 # For physical sub-camera: if RAW available, only use if not 4:3 or 16:9 254 use_raw_fov = False 255 if raw_avlb: 256 pixel_array_w = props['android.sensor.info.pixelArraySize']['width'] 257 pixel_array_h = props['android.sensor.info.pixelArraySize']['height'] 258 logging.debug('Pixel array size: %dx%d', pixel_array_w, pixel_array_h) 259 raw_aspect_ratio = pixel_array_w / pixel_array_h 260 use_raw_fov = ( 261 fls_physical == fls_logical or not 262 any(math.isclose(raw_aspect_ratio, jpeg_ar, abs_tol=_AR_DIFF_ATOL) 263 for jpeg_ar in _AR_FOR_JPEG_REFERENCE) 264 ) 265 266 ref_fov, cc_ct_gt, aspect_ratio_gt = ( 267 image_fov_utils.find_fov_reference( 268 cam, req, props, use_raw_fov, name_with_log_path)) 269 270 run_crop_test = full_or_better and raw_avlb 271 if run_crop_test: 272 # Normalize the circle size to 1/4 of the image size, so that 273 # circle size won't affect the crop test result 274 crop_thresh_factor = ((min(ref_fov['w'], ref_fov['h']) / 4.0) / 275 max(ref_fov['circle_w'], ref_fov['circle_h'])) 276 else: 277 logging.debug('Crop test skipped') 278 279 # Take pictures of each settings with all the image sizes available. 280 for fmt in format_list: 281 fmt_iter = fmt['iter'] 282 fmt_cmpr = fmt['cmpr'] 283 # Get the size of 'cmpr'. 284 sizes = capture_request_utils.get_available_output_sizes( 285 fmt_cmpr, props, fmt['cmpr_size']) 286 if not sizes: # Device might not support RAW. 287 continue 288 w_cmpr, h_cmpr = sizes[0][0], sizes[0][1] 289 for size_iter in capture_request_utils.get_available_output_sizes( 290 fmt_iter, props, fmt['iter_max']): 291 w_iter, h_iter = size_iter[0], size_iter[1] 292 # Skip same format/size combination: ITS doesn't handle that properly. 293 if w_iter*h_iter == w_cmpr*h_cmpr and fmt_iter == fmt_cmpr: 294 continue 295 out_surface = [{'width': w_iter, 'height': h_iter, 296 'format': fmt_iter}] 297 out_surface.append({'width': w_cmpr, 'height': h_cmpr, 298 'format': fmt_cmpr}) 299 cap = cam.do_capture(req, out_surface)[0] 300 _check_basic_correctness(cap, fmt_iter, w_iter, h_iter) 301 logging.debug('Captured %s with %s %dx%d. Compared size: %dx%d', 302 fmt_iter, fmt_cmpr, w_iter, h_iter, w_cmpr, h_cmpr) 303 img = image_processing_utils.convert_capture_to_rgb_image(cap) 304 img *= 255 # cv2 uses [0, 255]. 305 img_name = f'{name_with_log_path}_{fmt_iter}_with_{fmt_cmpr}_w{w_iter}_h{h_iter}.png' 306 circle = opencv_processing_utils.find_circle( 307 img, img_name, image_fov_utils.CIRCLE_MIN_AREA, 308 image_fov_utils.CIRCLE_COLOR) 309 if debug: 310 opencv_processing_utils.append_circle_center_to_img(circle, img, 311 img_name) 312 313 # Check pass/fail for fov coverage for all fmts in AR_CHECKED 314 img /= 255 # image_processing_utils uses [0, 1]. 315 if _is_checked_aspect_ratio(first_api_level, w_iter, h_iter): 316 fov_chk_msg = image_fov_utils.check_fov( 317 circle, ref_fov, w_iter, h_iter) 318 if fov_chk_msg: 319 failed_fov.append(fov_chk_msg) 320 image_processing_utils.write_image(img, img_name, True) 321 322 # Check pass/fail for aspect ratio. 323 ar_chk_msg = image_fov_utils.check_ar( 324 circle, aspect_ratio_gt, w_iter, h_iter, 325 f'{fmt_iter} with {fmt_cmpr}') 326 if ar_chk_msg: 327 failed_ar.append(ar_chk_msg) 328 image_processing_utils.write_image(img, img_name, True) 329 330 # Check pass/fail for crop. 331 if run_crop_test: 332 crop_chk_msg = image_fov_utils.check_crop( 333 circle, cc_ct_gt, w_iter, h_iter, 334 f'{fmt_iter} with {fmt_cmpr}', crop_thresh_factor) 335 if crop_chk_msg: 336 failed_crop.append(crop_chk_msg) 337 image_processing_utils.write_image(img, img_name, True) 338 339 # Print any failed test results. 340 _print_failed_test_results(failed_ar, failed_fov, failed_crop, 341 first_api_level, level3) 342 343if __name__ == '__main__': 344 test_runner.main() 345