1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Validate preview aspect ratio, crop and FoV vs format.""" 15 16import logging 17import os 18 19from mobly import test_runner 20 21import its_base_test 22import camera_properties_utils 23import capture_request_utils 24import image_fov_utils 25import image_processing_utils 26import its_session_utils 27import opencv_processing_utils 28import video_processing_utils 29 30 31_NAME = os.path.splitext(os.path.basename(__file__))[0] 32_VIDEO_DURATION = 3 # seconds 33_MAX_8BIT_IMGS = 255 34 35 36def _collect_data(cam, preview_size): 37 """Capture a preview video from the device. 38 39 Captures camera preview frames from the passed device. 40 41 Args: 42 cam: camera object 43 preview_size: str; preview resolution. ex. '1920x1080' 44 45 Returns: 46 recording object as described by cam.do_preview_recording 47 """ 48 49 recording_obj = cam.do_preview_recording(preview_size, _VIDEO_DURATION, False) 50 logging.debug('Recorded output path: %s', recording_obj['recordedOutputPath']) 51 logging.debug('Tested quality: %s', recording_obj['quality']) 52 53 return recording_obj 54 55 56def _print_failed_test_results(failed_ar, failed_fov, failed_crop): 57 """Print failed test results.""" 58 if failed_ar: 59 logging.error('Aspect ratio test summary') 60 logging.error('Images failed in the aspect ratio test:') 61 logging.error('Aspect ratio value: width / height') 62 for fa in failed_ar: 63 logging.error('%s', fa) 64 65 if failed_fov: 66 logging.error('FoV test summary') 67 logging.error('Images failed in the FoV test:') 68 for fov in failed_fov: 69 logging.error('%s', str(fov)) 70 71 if failed_crop: 72 logging.error('Crop test summary') 73 logging.error('Images failed in the crop test:') 74 logging.error('Circle center (H x V) relative to the image center.') 75 for fc in failed_crop: 76 logging.error('%s', fc) 77 78 79class PreviewAspectRatioAndCropTest(its_base_test.ItsBaseTest): 80 """Test preview aspect ratio/field of view/cropping for each tested fmt. 81 82 This test checks for: 83 1. Aspect ratio: images are not stretched 84 2. Crop: center of images is not shifted 85 3. FOV: images cropped to keep maximum possible FOV with only 1 dimension 86 (horizontal or veritical) cropped. 87 88 89 90 The test preview is a black circle on a white background. 91 92 When RAW capture is available, set the height vs. width ratio of the circle in 93 the full-frame RAW as ground truth. In an ideal setup such ratio should be 94 very close to 1.0, but here we just use the value derived from full resolution 95 RAW as ground truth to account for the possibility that the chart is not well 96 positioned to be precisely parallel to image sensor plane. 97 The test then compares the ground truth ratio with the same ratio measured 98 on previews captured using different formats. 99 100 If RAW capture is unavailable, a full resolution JPEG image is used to setup 101 ground truth. In this case, the ground truth aspect ratio is defined as 1.0 102 and it is the tester's responsibility to make sure the test chart is 103 properly positioned so the detected circles indeed have aspect ratio close 104 to 1.0 assuming no bugs causing image stretched. 105 106 The aspect ratio test checks the aspect ratio of the detected circle and 107 it will fail if the aspect ratio differs too much from the ground truth 108 aspect ratio mentioned above. 109 110 The FOV test examines the ratio between the detected circle area and the 111 image size. When the aspect ratio of the test image is the same as the 112 ground truth image, the ratio should be very close to the ground truth 113 value. When the aspect ratio is different, the difference is factored in 114 per the expectation of the Camera2 API specification, which mandates the 115 FOV reduction from full sensor area must only occur in one dimension: 116 horizontally or vertically, and never both. For example, let's say a sensor 117 has a 16:10 full sensor FOV. For all 16:10 output images there should be no 118 FOV reduction on them. For 16:9 output images the FOV should be vertically 119 cropped by 9/10. For 4:3 output images the FOV should be cropped 120 horizontally instead and the ratio (r) can be calculated as follows: 121 (16 * r) / 10 = 4 / 3 => r = 40 / 48 = 0.8333 122 Say the circle is covering x percent of the 16:10 sensor on the full 16:10 123 FOV, and assume the circle in the center will never be cut in any output 124 sizes (this can be achieved by picking the right size and position of the 125 test circle), the from above cropping expectation we can derive on a 16:9 126 output image the circle will cover (x / 0.9) percent of the 16:9 image; on 127 a 4:3 output image the circle will cover (x / 0.8333) percent of the 4:3 128 image. 129 130 The crop test checks that the center of any output image remains aligned 131 with center of sensor's active area, no matter what kind of cropping or 132 scaling is applied. The test verifies that by checking the relative vector 133 from the image center to the center of detected circle remains unchanged. 134 The relative part is normalized by the detected circle size to account for 135 scaling effect. 136 """ 137 138 def test_preview_aspect_ratio_and_crop(self): 139 log_path = self.log_path 140 video_processing_utils.log_ffmpeg_version() 141 142 with its_session_utils.ItsSession( 143 device_id=self.dut.serial, 144 camera_id=self.camera_id, 145 hidden_physical_id=self.hidden_physical_id) as cam: 146 failed_ar = [] # Streams failed the aspect ratio test 147 failed_crop = [] # Streams failed the crop test 148 failed_fov = [] # Streams that fail FoV test 149 props = cam.get_camera_properties() 150 fls_logical = props['android.lens.info.availableFocalLengths'] 151 logging.debug('logical available focal lengths: %s', str(fls_logical)) 152 props = cam.override_with_hidden_physical_camera_props(props) 153 fls_physical = props['android.lens.info.availableFocalLengths'] 154 logging.debug('physical available focal lengths: %s', str(fls_physical)) 155 name_with_log_path = f'{os.path.join(self.log_path, _NAME)}' 156 157 # Check SKIP conditions 158 first_api_level = its_session_utils.get_first_api_level(self.dut.serial) 159 camera_properties_utils.skip_unless( 160 first_api_level >= its_session_utils.ANDROID14_API_LEVEL) 161 162 # Load scene 163 its_session_utils.load_scene(cam, props, self.scene, 164 self.tablet, self.chart_distance) 165 # Raise error if not FRONT or REAR facing camera 166 facing = props['android.lens.facing'] 167 if (facing != camera_properties_utils.LENS_FACING_BACK 168 and facing != camera_properties_utils.LENS_FACING_FRONT): 169 raise AssertionError('Unknown lens facing: {facing}.') 170 171 # List of preview resolutions to test 172 supported_preview_sizes = cam.get_supported_preview_sizes(self.camera_id) 173 for size in video_processing_utils.LOW_RESOLUTION_SIZES: 174 if size in supported_preview_sizes: 175 supported_preview_sizes.remove(size) 176 logging.debug('Supported preview resolutions: %s', 177 supported_preview_sizes) 178 raw_avlb = camera_properties_utils.raw16(props) 179 full_or_better = camera_properties_utils.full_or_better(props) 180 debug = self.debug_mode 181 182 # Converge 3A 183 cam.do_3a() 184 req = capture_request_utils.auto_capture_request() 185 if raw_avlb and (fls_physical == fls_logical): 186 logging.debug('RAW') 187 raw_bool = True 188 else: 189 logging.debug('JPEG') 190 raw_bool = False 191 ref_fov, cc_ct_gt, aspect_ratio_gt = image_fov_utils.find_fov_reference( 192 cam, req, props, raw_bool, name_with_log_path) 193 194 run_crop_test = full_or_better and raw_avlb 195 for preview_size in supported_preview_sizes: 196 quality = preview_size.split(':')[0] 197 198 # Check if we support testing this quality 199 if quality in video_processing_utils.ITS_SUPPORTED_QUALITIES: 200 logging.debug('Testing preview recording for quality: %s', quality) 201 # recording preview 202 preview_rec_obj = _collect_data(cam, preview_size) 203 204 # Grab the recording from DUT 205 self.dut.adb.pull([preview_rec_obj['recordedOutputPath'], log_path]) 206 preview_file_name = (preview_rec_obj['recordedOutputPath'] 207 .split('/')[-1]) 208 logging.debug('preview_file_name: %s', preview_file_name) 209 preview_size = preview_rec_obj['videoSize'] 210 width = int(preview_size.split('x')[0]) 211 height = int(preview_size.split('x')[-1]) 212 213 key_frame_files = [] 214 key_frame_files = ( 215 video_processing_utils.extract_key_frames_from_video( 216 self.log_path, preview_file_name) 217 ) 218 logging.debug('key_frame_files: %s', key_frame_files) 219 220 # Get the key frame file to process 221 last_key_frame_file = ( 222 video_processing_utils.get_key_frame_to_process(key_frame_files) 223 ) 224 logging.debug('last_key_frame: %s', last_key_frame_file) 225 last_key_frame_path = os.path.join( 226 self.log_path, last_key_frame_file) 227 228 # Convert lastKeyFrame to numpy array 229 np_image = image_processing_utils.convert_image_to_numpy_array( 230 last_key_frame_path) 231 logging.debug('numpy image shape: %s', np_image.shape) 232 233 # Check fov 234 ref_img_name = (f'{name_with_log_path}_{quality}' 235 f'_w{width}_h{height}_circle.png') 236 circle = opencv_processing_utils.find_circle( 237 np_image, ref_img_name, image_fov_utils.CIRCLE_MIN_AREA, 238 image_fov_utils.CIRCLE_COLOR) 239 240 if debug: 241 opencv_processing_utils.append_circle_center_to_img( 242 circle, np_image, ref_img_name) 243 244 max_img_value = _MAX_8BIT_IMGS 245 246 # Check pass/fail for fov coverage for all fmts in AR_CHECKED 247 img_name_stem = f'{name_with_log_path}_{quality}_w{width}_h{height}' 248 fov_chk_msg = image_fov_utils.check_fov( 249 circle, ref_fov, width, height) 250 if fov_chk_msg: 251 img_name = f'{img_name_stem}_fov.png' 252 fov_chk_quality_msg = f'Quality: {quality} {fov_chk_msg}' 253 failed_fov.append(fov_chk_quality_msg) 254 image_processing_utils.write_image( 255 np_image/max_img_value, img_name, True) 256 257 # Check pass/fail for aspect ratio 258 ar_chk_msg = image_fov_utils.check_ar( 259 circle, aspect_ratio_gt, width, height, 260 f'{quality}') 261 if ar_chk_msg: 262 img_name = f'{img_name_stem}_ar.png' 263 failed_ar.append(ar_chk_msg) 264 image_processing_utils.write_image( 265 np_image/max_img_value, img_name, True) 266 267 # Check pass/fail for crop 268 if run_crop_test: 269 # Normalize the circle size to 1/4 of the image size, so that 270 # circle size won't affect the crop test result 271 crop_thresh_factor = ((min(ref_fov['w'], ref_fov['h']) / 4.0) / 272 max(ref_fov['circle_w'], 273 ref_fov['circle_h'])) 274 crop_chk_msg = image_fov_utils.check_crop( 275 circle, cc_ct_gt, width, height, 276 f'{quality}', crop_thresh_factor) 277 if crop_chk_msg: 278 crop_img_name = f'{img_name_stem}_crop.png' 279 failed_crop.append(crop_chk_msg) 280 image_processing_utils.write_image(np_image/max_img_value, 281 crop_img_name, True) 282 else: 283 logging.debug('Crop test skipped') 284 285 # Print any failed test results 286 _print_failed_test_results(failed_ar, failed_fov, failed_crop) 287 288 e_msg = '' 289 if failed_ar: 290 e_msg = 'Aspect ratio ' 291 if failed_fov: 292 e_msg += 'FoV ' 293 if failed_crop: 294 e_msg += 'Crop ' 295 if e_msg: 296 raise AssertionError(f'{e_msg}check failed.') 297 298if __name__ == '__main__': 299 test_runner.main() 300