1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Validate video aspect ratio, crop and FoV vs format.""" 15 16import logging 17import math 18import os.path 19 20from mobly import test_runner 21 22import its_base_test 23import camera_properties_utils 24import capture_request_utils 25import image_fov_utils 26import image_processing_utils 27import its_session_utils 28import opencv_processing_utils 29import video_processing_utils 30 31_NAME = os.path.splitext(os.path.basename(__file__))[0] 32_VIDEO_RECORDING_DURATION_SECONDS = 3 33_FOV_PERCENT_RTOL = 0.15 # Relative tolerance on circle FoV % to expected. 34_AR_CHECKED_PRE_API_30 = ('4:3', '16:9', '18:9') 35_AR_DIFF_ATOL = 0.01 36_AR_FOR_JPEG_REFERENCE = (4/3, 16/9) 37_MAX_8BIT_IMGS = 255 38_MAX_10BIT_IMGS = 1023 39 40 41def _print_failed_test_results(failed_ar, failed_fov, failed_crop, quality): 42 """Print failed test results.""" 43 if failed_ar: 44 logging.error('Aspect ratio test summary for quality: %s', quality) 45 logging.error('Images failed in the aspect ratio test:') 46 logging.error('Aspect ratio value: width / height') 47 for fa in failed_ar: 48 logging.error('%s', fa) 49 50 if failed_fov: 51 logging.error('FoV test summary for quality: %s', quality) 52 logging.error('Images failed in the FoV test:') 53 for fov in failed_fov: 54 logging.error('%s', str(fov)) 55 56 if failed_crop: 57 logging.error('Crop test summary for quality: %s', quality) 58 logging.error('Images failed in the crop test:') 59 logging.error('Circle center (H x V) relative to the image center.') 60 for fc in failed_crop: 61 logging.error('%s', fc) 62 63 64class VideoAspectRatioAndCropTest(its_base_test.ItsBaseTest): 65 """Test aspect ratio/field of view/cropping for each tested fmt. 66 67 This test checks for: 68 1. Aspect ratio: images are not stretched 69 2. Crop: center of images is not shifted 70 3. FOV: images cropped to keep maximum possible FOV with only 1 dimension 71 (horizontal or veritical) cropped. 72 73 Video recording will be done using the SDR profile as well as HLG10 74 if available. 75 76 The test video is a black circle on a white background. 77 78 When RAW capture is available, set the height vs. width ratio of the circle in 79 the full-frame RAW as ground truth. In an ideal setup such ratio should be 80 very close to 1.0, but here we just use the value derived from full resolution 81 RAW as ground truth to account for the possibility that the chart is not 82 well positioned to be precisely parallel to image sensor plane. 83 The test then compares the ground truth ratio with the same ratio measured 84 on videos captured using different formats. 85 86 If RAW capture is unavailable, a full resolution JPEG image is used to setup 87 ground truth. In this case, the ground truth aspect ratio is defined as 1.0 88 and it is the tester's responsibility to make sure the test chart is 89 properly positioned so the detected circles indeed have aspect ratio close 90 to 1.0 assuming no bugs causing image stretched. 91 92 The aspect ratio test checks the aspect ratio of the detected circle and 93 it will fail if the aspect ratio differs too much from the ground truth 94 aspect ratio mentioned above. 95 96 The FOV test examines the ratio between the detected circle area and the 97 image size. When the aspect ratio of the test image is the same as the 98 ground truth image, the ratio should be very close to the ground truth 99 value. When the aspect ratio is different, the difference is factored in 100 per the expectation of the Camera2 API specification, which mandates the 101 FOV reduction from full sensor area must only occur in one dimension: 102 horizontally or vertically, and never both. For example, let's say a sensor 103 has a 16:10 full sensor FOV. For all 16:10 output images there should be no 104 FOV reduction on them. For 16:9 output images the FOV should be vertically 105 cropped by 9/10. For 4:3 output images the FOV should be cropped 106 horizontally instead and the ratio (r) can be calculated as follows: 107 (16 * r) / 10 = 4 / 3 => r = 40 / 48 = 0.8333 108 Say the circle is covering x percent of the 16:10 sensor on the full 16:10 109 FOV, and assume the circle in the center will never be cut in any output 110 sizes (this can be achieved by picking the right size and position of the 111 test circle), the from above cropping expectation we can derive on a 16:9 112 output image the circle will cover (x / 0.9) percent of the 16:9 image; on 113 a 4:3 output image the circle will cover (x / 0.8333) percent of the 4:3 114 image. 115 116 The crop test checks that the center of any output image remains aligned 117 with center of sensor's active area, no matter what kind of cropping or 118 scaling is applied. The test verifies that by checking the relative vector 119 from the image center to the center of detected circle remains unchanged. 120 The relative part is normalized by the detected circle size to account for 121 scaling effect. 122 """ 123 124 def test_video_aspect_ratio_and_crop(self): 125 logging.debug('Starting %s', _NAME) 126 failed_ar = [] # Streams failed the aspect ratio test. 127 failed_crop = [] # Streams failed the crop test. 128 failed_fov = [] # Streams that fail FoV test. 129 130 with its_session_utils.ItsSession( 131 device_id=self.dut.serial, 132 camera_id=self.camera_id, 133 hidden_physical_id=self.hidden_physical_id) as cam: 134 props = cam.get_camera_properties() 135 fls_logical = props['android.lens.info.availableFocalLengths'] 136 logging.debug('logical available focal lengths: %s', str(fls_logical)) 137 props = cam.override_with_hidden_physical_camera_props(props) 138 fls_physical = props['android.lens.info.availableFocalLengths'] 139 logging.debug('physical available focal lengths: %s', str(fls_physical)) 140 name_with_log_path = os.path.join(self.log_path, _NAME) 141 142 # Check SKIP conditions. 143 first_api_level = its_session_utils.get_first_api_level(self.dut.serial) 144 camera_properties_utils.skip_unless( 145 first_api_level >= its_session_utils.ANDROID13_API_LEVEL) 146 147 # Load scene. 148 its_session_utils.load_scene(cam, props, self.scene, 149 self.tablet, self.chart_distance) 150 151 # Determine camera capabilities. 152 supported_video_qualities = cam.get_supported_video_qualities( 153 self.camera_id) 154 logging.debug('Supported video qualities: %s', supported_video_qualities) 155 full_or_better = camera_properties_utils.full_or_better(props) 156 raw_avlb = camera_properties_utils.raw16(props) 157 debug = self.debug_mode 158 159 # Converge 3A. 160 cam.do_3a() 161 req = capture_request_utils.auto_capture_request() 162 163 # For main camera: if RAW available, use it as ground truth, else JPEG 164 # For physical sub-camera: if RAW available, only use if not 4:3 or 16:9 165 if raw_avlb: 166 pixel_array_w = props['android.sensor.info.pixelArraySize']['width'] 167 pixel_array_h = props['android.sensor.info.pixelArraySize']['height'] 168 logging.debug('Pixel array size: %dx%d', pixel_array_w, pixel_array_h) 169 raw_aspect_ratio = pixel_array_w / pixel_array_h 170 if (fls_physical == fls_logical or not 171 any(math.isclose(raw_aspect_ratio, jpeg_ar, abs_tol=_AR_DIFF_ATOL) 172 for jpeg_ar in _AR_FOR_JPEG_REFERENCE)): 173 logging.debug('RAW') 174 use_raw_fov = True 175 else: 176 logging.debug('RAW available, but using JPEG as ground truth') 177 use_raw_fov = False 178 else: 179 logging.debug('JPEG') 180 use_raw_fov = False 181 182 ref_fov, cc_ct_gt, aspect_ratio_gt = image_fov_utils.find_fov_reference( 183 cam, req, props, use_raw_fov, name_with_log_path) 184 185 run_crop_test = full_or_better and raw_avlb 186 187 # Log ffmpeg version being used. 188 video_processing_utils.log_ffmpeg_version() 189 190 for quality_profile_id_pair in supported_video_qualities: 191 quality = quality_profile_id_pair.split(':')[0] 192 profile_id = quality_profile_id_pair.split(':')[-1] 193 # Check if we support testing this quality. 194 if quality in video_processing_utils.ITS_SUPPORTED_QUALITIES: 195 logging.debug('Testing video recording for quality: %s', quality) 196 hlg10_params = [False] 197 hlg10_supported = cam.is_hlg10_recording_supported(profile_id) 198 logging.debug('HLG10 supported: %s', hlg10_supported) 199 if hlg10_supported: 200 hlg10_params.append(hlg10_supported) 201 202 for hlg10_param in hlg10_params: 203 video_recording_obj = cam.do_basic_recording( 204 profile_id, quality, _VIDEO_RECORDING_DURATION_SECONDS, 0, 205 hlg10_param) 206 logging.debug('video_recording_obj: %s', video_recording_obj) 207 # TODO(ruchamk): Modify video recording object to send videoFrame 208 # width and height instead of videoSize to avoid string operation 209 # here. 210 video_size = video_recording_obj['videoSize'] 211 width = int(video_size.split('x')[0]) 212 height = int(video_size.split('x')[-1]) 213 214 # Pull the video recording file from the device. 215 self.dut.adb.pull([video_recording_obj['recordedOutputPath'], 216 self.log_path]) 217 logging.debug('Recorded video is available at: %s', 218 self.log_path) 219 video_file_name = video_recording_obj[ 220 'recordedOutputPath'].split('/')[-1] 221 logging.debug('video_file_name: %s', video_file_name) 222 223 key_frame_files = [] 224 key_frame_files = ( 225 video_processing_utils.extract_key_frames_from_video( 226 self.log_path, video_file_name) 227 ) 228 logging.debug('key_frame_files:%s', key_frame_files) 229 230 # Get the key frame file to process. 231 last_key_frame_file = ( 232 video_processing_utils.get_key_frame_to_process( 233 key_frame_files) 234 ) 235 logging.debug('last_key_frame: %s', last_key_frame_file) 236 last_key_frame_path = os.path.join( 237 self.log_path, last_key_frame_file) 238 239 # Convert lastKeyFrame to numpy array 240 np_image = image_processing_utils.convert_image_to_numpy_array( 241 last_key_frame_path) 242 logging.debug('numpy image shape: %s', np_image.shape) 243 244 # Check fov 245 ref_img_name = (f'{name_with_log_path}_{quality}' 246 f'_w{width}_h{height}_circle.png') 247 circle = opencv_processing_utils.find_circle( 248 np_image, ref_img_name, image_fov_utils.CIRCLE_MIN_AREA, 249 image_fov_utils.CIRCLE_COLOR) 250 251 if debug: 252 opencv_processing_utils.append_circle_center_to_img( 253 circle, np_image, ref_img_name) 254 255 max_img_value = _MAX_8BIT_IMGS 256 if hlg10_param: 257 max_img_value = _MAX_10BIT_IMGS 258 259 # Check pass/fail for fov coverage for all fmts in AR_CHECKED 260 img_name_stem = f'{name_with_log_path}_{quality}_w{width}_h{height}' 261 fov_chk_msg = image_fov_utils.check_fov( 262 circle, ref_fov, width, height) 263 if fov_chk_msg: 264 img_name = f'{img_name_stem}_fov.png' 265 fov_chk_quality_msg = f'Quality: {quality} {fov_chk_msg}' 266 failed_fov.append(fov_chk_quality_msg) 267 image_processing_utils.write_image( 268 np_image/max_img_value, img_name, True) 269 270 # Check pass/fail for aspect ratio. 271 ar_chk_msg = image_fov_utils.check_ar( 272 circle, aspect_ratio_gt, width, height, 273 f'{quality}') 274 if ar_chk_msg: 275 img_name = f'{img_name_stem}_ar.png' 276 failed_ar.append(ar_chk_msg) 277 image_processing_utils.write_image( 278 np_image/max_img_value, img_name, True) 279 280 # Check pass/fail for crop. 281 if run_crop_test: 282 # Normalize the circle size to 1/4 of the image size, so that 283 # circle size won't affect the crop test result 284 crop_thresh_factor = ((min(ref_fov['w'], ref_fov['h']) / 4.0) / 285 max(ref_fov['circle_w'], 286 ref_fov['circle_h'])) 287 crop_chk_msg = image_fov_utils.check_crop( 288 circle, cc_ct_gt, width, height, 289 f'{quality}', crop_thresh_factor) 290 if crop_chk_msg: 291 crop_img_name = f'{img_name_stem}_crop.png' 292 failed_crop.append(crop_chk_msg) 293 image_processing_utils.write_image(np_image/max_img_value, 294 crop_img_name, True) 295 else: 296 logging.debug('Crop test skipped') 297 298 # Print any failed test results. 299 _print_failed_test_results(failed_ar, failed_fov, failed_crop, quality) 300 e_msg = '' 301 if failed_ar: 302 e_msg = 'Aspect ratio ' 303 if failed_fov: 304 e_msg += 'FoV ' 305 if failed_crop: 306 e_msg += 'Crop ' 307 if e_msg: 308 raise AssertionError(f'{e_msg}check failed.') 309 310if __name__ == '__main__': 311 test_runner.main() 312