1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Validate video aspect ratio, crop and FoV vs format.""" 15 16import logging 17import math 18import os.path 19 20from mobly import test_runner 21 22import its_base_test 23import camera_properties_utils 24import capture_request_utils 25import image_fov_utils 26import image_processing_utils 27import its_session_utils 28import opencv_processing_utils 29import video_processing_utils 30 31_AR_CHECKED_PRE_API_30 = ('4:3', '16:9', '18:9') 32_AR_DIFF_ATOL = 0.01 33_AR_FOR_JPEG_REFERENCE = (4/3, 16/9) 34_FOV_PERCENT_RTOL = 0.15 # Relative tolerance on circle FoV % to expected. 35_HGL10_TESTED_QUALITIES = ('HIGH', '480P', '720P', '1080P', '2160P') 36_MAX_8BIT_IMGS = 255 37_MAX_10BIT_IMGS = 1023 38_NAME = os.path.splitext(os.path.basename(__file__))[0] 39_VIDEO_RECORDING_DURATION_SECONDS = 3 40 41 42def _print_failed_test_results(failed_ar, failed_fov, failed_crop, quality): 43 """Print failed test results.""" 44 if failed_ar: 45 logging.error('Aspect ratio test summary for quality: %s', quality) 46 logging.error('Images failed in the aspect ratio test:') 47 logging.error('Aspect ratio value: width / height') 48 for fa in failed_ar: 49 logging.error('%s', fa) 50 51 if failed_fov: 52 logging.error('FoV test summary for quality: %s', quality) 53 logging.error('Images failed in the FoV test:') 54 for fov in failed_fov: 55 logging.error('%s', str(fov)) 56 57 if failed_crop: 58 logging.error('Crop test summary for quality: %s', quality) 59 logging.error('Images failed in the crop test:') 60 logging.error('Circle center (H x V) relative to the image center.') 61 for fc in failed_crop: 62 logging.error('%s', fc) 63 64 65class VideoAspectRatioAndCropTest(its_base_test.ItsBaseTest): 66 """Test aspect ratio/field of view/cropping for each tested fmt. 67 68 This test checks for: 69 1. Aspect ratio: images are not stretched 70 2. Crop: center of images is not shifted 71 3. FOV: images cropped to keep maximum possible FOV with only 1 dimension 72 (horizontal or veritical) cropped. 73 74 Video recording will be done using the SDR profile as well as HLG10 75 if available. 76 77 The test video is a black circle on a white background. 78 79 When RAW capture is available, set the height vs. width ratio of the circle in 80 the full-frame RAW as ground truth. In an ideal setup such ratio should be 81 very close to 1.0, but here we just use the value derived from full resolution 82 RAW as ground truth to account for the possibility that the chart is not 83 well positioned to be precisely parallel to image sensor plane. 84 The test then compares the ground truth ratio with the same ratio measured 85 on videos captured using different formats. 86 87 If RAW capture is unavailable, a full resolution JPEG image is used to setup 88 ground truth. In this case, the ground truth aspect ratio is defined as 1.0 89 and it is the tester's responsibility to make sure the test chart is 90 properly positioned so the detected circles indeed have aspect ratio close 91 to 1.0 assuming no bugs causing image stretched. 92 93 The aspect ratio test checks the aspect ratio of the detected circle and 94 it will fail if the aspect ratio differs too much from the ground truth 95 aspect ratio mentioned above. 96 97 The FOV test examines the ratio between the detected circle area and the 98 image size. When the aspect ratio of the test image is the same as the 99 ground truth image, the ratio should be very close to the ground truth 100 value. When the aspect ratio is different, the difference is factored in 101 per the expectation of the Camera2 API specification, which mandates the 102 FOV reduction from full sensor area must only occur in one dimension: 103 horizontally or vertically, and never both. For example, let's say a sensor 104 has a 16:10 full sensor FOV. For all 16:10 output images there should be no 105 FOV reduction on them. For 16:9 output images the FOV should be vertically 106 cropped by 9/10. For 4:3 output images the FOV should be cropped 107 horizontally instead and the ratio (r) can be calculated as follows: 108 (16 * r) / 10 = 4 / 3 => r = 40 / 48 = 0.8333 109 Say the circle is covering x percent of the 16:10 sensor on the full 16:10 110 FOV, and assume the circle in the center will never be cut in any output 111 sizes (this can be achieved by picking the right size and position of the 112 test circle), the from above cropping expectation we can derive on a 16:9 113 output image the circle will cover (x / 0.9) percent of the 16:9 image; on 114 a 4:3 output image the circle will cover (x / 0.8333) percent of the 4:3 115 image. 116 117 The crop test checks that the center of any output image remains aligned 118 with center of sensor's active area, no matter what kind of cropping or 119 scaling is applied. The test verifies that by checking the relative vector 120 from the image center to the center of detected circle remains unchanged. 121 The relative part is normalized by the detected circle size to account for 122 scaling effect. 123 """ 124 125 def test_video_aspect_ratio_and_crop(self): 126 logging.debug('Starting %s', _NAME) 127 failed_ar = [] # Streams failed the aspect ratio test. 128 failed_crop = [] # Streams failed the crop test. 129 failed_fov = [] # Streams that fail FoV test. 130 131 with its_session_utils.ItsSession( 132 device_id=self.dut.serial, 133 camera_id=self.camera_id, 134 hidden_physical_id=self.hidden_physical_id) as cam: 135 props = cam.get_camera_properties() 136 fls_logical = props['android.lens.info.availableFocalLengths'] 137 logging.debug('logical available focal lengths: %s', str(fls_logical)) 138 props = cam.override_with_hidden_physical_camera_props(props) 139 fls_physical = props['android.lens.info.availableFocalLengths'] 140 logging.debug('physical available focal lengths: %s', str(fls_physical)) 141 name_with_log_path = os.path.join(self.log_path, _NAME) 142 143 # Check SKIP conditions. 144 first_api_level = its_session_utils.get_first_api_level(self.dut.serial) 145 camera_properties_utils.skip_unless( 146 first_api_level >= its_session_utils.ANDROID13_API_LEVEL) 147 148 # Load scene. 149 its_session_utils.load_scene(cam, props, self.scene, 150 self.tablet, self.chart_distance) 151 152 # Determine camera capabilities. 153 supported_video_qualities = cam.get_supported_video_qualities( 154 self.camera_id) 155 logging.debug('Supported video qualities: %s', supported_video_qualities) 156 full_or_better = camera_properties_utils.full_or_better(props) 157 raw_avlb = camera_properties_utils.raw16(props) 158 159 # Converge 3A. 160 cam.do_3a() 161 req = capture_request_utils.auto_capture_request() 162 163 # For main camera: if RAW available, use it as ground truth, else JPEG 164 # For physical sub-camera: if RAW available, only use if not 4:3 or 16:9 165 if raw_avlb: 166 pixel_array_w = props['android.sensor.info.pixelArraySize']['width'] 167 pixel_array_h = props['android.sensor.info.pixelArraySize']['height'] 168 logging.debug('Pixel array size: %dx%d', pixel_array_w, pixel_array_h) 169 raw_aspect_ratio = pixel_array_w / pixel_array_h 170 if (fls_physical == fls_logical or not 171 any(math.isclose(raw_aspect_ratio, jpeg_ar, abs_tol=_AR_DIFF_ATOL) 172 for jpeg_ar in _AR_FOR_JPEG_REFERENCE)): 173 logging.debug('RAW') 174 use_raw_fov = True 175 else: 176 logging.debug('RAW available, but using JPEG as ground truth') 177 use_raw_fov = False 178 else: 179 logging.debug('JPEG') 180 use_raw_fov = False 181 182 ref_fov, cc_ct_gt, aspect_ratio_gt = image_fov_utils.find_fov_reference( 183 cam, req, props, use_raw_fov, name_with_log_path) 184 185 run_crop_test = full_or_better and raw_avlb 186 187 # Log ffmpeg version being used. 188 video_processing_utils.log_ffmpeg_version() 189 190 for quality_profile_id_pair in supported_video_qualities: 191 quality = quality_profile_id_pair.split(':')[0] 192 profile_id = quality_profile_id_pair.split(':')[-1] 193 # Check if we support testing this quality. 194 if quality in video_processing_utils.ITS_SUPPORTED_QUALITIES: 195 logging.debug('Testing video recording for quality: %s', quality) 196 hlg10_params = [False] 197 hlg10_supported = cam.is_hlg10_recording_supported_for_profile( 198 profile_id) 199 logging.debug('HLG10 supported: %s', hlg10_supported) 200 if hlg10_supported and quality in _HGL10_TESTED_QUALITIES: 201 hlg10_params.append(hlg10_supported) 202 203 for hlg10_param in hlg10_params: 204 video_recording_obj = cam.do_basic_recording( 205 profile_id, quality, _VIDEO_RECORDING_DURATION_SECONDS, 0, 206 hlg10_param) 207 logging.debug('video_recording_obj: %s', video_recording_obj) 208 # TODO(ruchamk): Modify video recording object to send videoFrame 209 # width and height instead of videoSize to avoid string operation 210 # here. 211 video_size = video_recording_obj['videoSize'] 212 width = int(video_size.split('x')[0]) 213 height = int(video_size.split('x')[-1]) 214 215 # Pull the video recording file from the device. 216 self.dut.adb.pull([video_recording_obj['recordedOutputPath'], 217 self.log_path]) 218 logging.debug('Recorded video is available at: %s', 219 self.log_path) 220 video_file_name = video_recording_obj[ 221 'recordedOutputPath'].split('/')[-1] 222 logging.debug('video_file_name: %s', video_file_name) 223 224 # Validate colorspace 225 colorspace = video_processing_utils.get_video_colorspace( 226 self.log_path, video_file_name) 227 if (hlg10_param and 228 video_processing_utils.COLORSPACE_HDR not in colorspace): 229 raise AssertionError('colorspace check failed for HDR.') 230 logging.debug('Colorspace test passed, video colorspace is %s', 231 colorspace) 232 233 # Extract last key frame as numpy image 234 last_key_frame = ( 235 video_processing_utils.extract_last_key_frame_from_recording( 236 self.log_path, video_file_name) 237 ) 238 239 # Check FoV 240 ref_img_name = (f'{name_with_log_path}_{quality}' 241 f'_w{width}_h{height}_circle.png') 242 circle = opencv_processing_utils.find_circle( 243 last_key_frame, ref_img_name, image_fov_utils.CIRCLE_MIN_AREA, 244 image_fov_utils.CIRCLE_COLOR) 245 246 opencv_processing_utils.append_circle_center_to_img( 247 circle, last_key_frame, ref_img_name) 248 249 max_img_value = _MAX_8BIT_IMGS 250 if hlg10_param: 251 max_img_value = _MAX_10BIT_IMGS 252 253 # Check pass/fail for fov coverage for all fmts in AR_CHECKED 254 img_name_stem = f'{name_with_log_path}_{quality}_w{width}_h{height}' 255 fov_chk_msg = image_fov_utils.check_fov( 256 circle, ref_fov, width, height) 257 if fov_chk_msg: 258 img_name = f'{img_name_stem}_fov.png' 259 fov_chk_quality_msg = f'Quality: {quality} {fov_chk_msg}' 260 failed_fov.append(fov_chk_quality_msg) 261 image_processing_utils.write_image( 262 last_key_frame/max_img_value, img_name, True) 263 264 # Check pass/fail for aspect ratio. 265 ar_chk_msg = image_fov_utils.check_ar( 266 circle, aspect_ratio_gt, width, height, 267 f'{quality}') 268 if ar_chk_msg: 269 img_name = f'{img_name_stem}_ar.png' 270 failed_ar.append(ar_chk_msg) 271 image_processing_utils.write_image( 272 last_key_frame/max_img_value, img_name, True) 273 274 # Check pass/fail for crop. 275 if run_crop_test: 276 # Normalize the circle size to 1/4 of the image size, so that 277 # circle size won't affect the crop test result 278 crop_thresh_factor = ((min(ref_fov['w'], ref_fov['h']) / 4.0) / 279 max(ref_fov['circle_w'], 280 ref_fov['circle_h'])) 281 crop_chk_msg = image_fov_utils.check_crop( 282 circle, cc_ct_gt, width, height, 283 f'{quality}', crop_thresh_factor) 284 if crop_chk_msg: 285 crop_img_name = f'{img_name_stem}_crop.png' 286 failed_crop.append(crop_chk_msg) 287 image_processing_utils.write_image(last_key_frame/max_img_value, 288 crop_img_name, True) 289 else: 290 logging.debug('Crop test skipped') 291 292 # Print any failed test results. 293 _print_failed_test_results(failed_ar, failed_fov, failed_crop, quality) 294 e_msg = '' 295 if failed_ar: 296 e_msg = 'Aspect ratio ' 297 if failed_fov: 298 e_msg += 'FoV ' 299 if failed_crop: 300 e_msg += 'Crop ' 301 if e_msg: 302 raise AssertionError(f'{e_msg}check failed.') 303 304if __name__ == '__main__': 305 test_runner.main() 306