1# Copyright 2023 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Verify preview matches video output during video zoom.""" 15 16import logging 17import math 18import multiprocessing 19import os 20import time 21 22import cv2 23from mobly import test_runner 24import numpy as np 25 26import its_base_test 27import camera_properties_utils 28import capture_request_utils 29import image_processing_utils 30import its_session_utils 31import opencv_processing_utils 32import video_processing_utils 33 34_CIRCLE_AR_RTOL = 0.15 # contour width vs height (aspect ratio) 35_CIRCLE_COLOR = 0 # [0: black, 255: white] 36_CIRCLE_R = 2 37_CIRCLE_X = 0 38_CIRCLE_Y = 1 39_CIRCLISH_RTOL = 0.15 # contour area vs ideal circle area pi*((w+h)/4)**2 40_LENS_FACING_FRONT = 0 41_LINE_COLOR = (255, 0, 0) # red 42_MAX_STR = 'max' 43_MIN_STR = 'min' 44_MIN_AREA_RATIO = 0.00015 # based on 2000/(4000x3000) pixels 45_MIN_CIRCLE_PTS = 25 46_MIN_ZOOM_CHART_SCALING = 0.7 47_MIN_SIZE = 1280*720 # 720P 48_NAME = os.path.splitext(os.path.basename(__file__))[0] 49_OFFSET_TOL = 5 # pixels 50_RADIUS_RTOL = 0.1 # 10% tolerance Video/Preview circle size 51_RECORDING_DURATION = 2 # seconds 52_ZOOM_COMP_MAX_THRESH = 1.15 53_ZOOM_MIN_THRESH = 2.0 54_ZOOM_RATIO = 2 55 56 57def _extract_key_frame_from_recording(log_path, file_name): 58 """Extract key frames from recordings. 59 60 Args: 61 log_path: str; file location 62 file_name: str file name for saved video 63 64 Returns: 65 dictionary of images 66 """ 67 key_frame_files = [] 68 key_frame_files = ( 69 video_processing_utils.extract_key_frames_from_video( 70 log_path, file_name) 71 ) 72 logging.debug('key_frame_files: %s', key_frame_files) 73 74 # Get the key frame file to process. 75 last_key_frame_file = ( 76 video_processing_utils.get_key_frame_to_process( 77 key_frame_files) 78 ) 79 logging.debug('last_key_frame: %s', last_key_frame_file) 80 last_key_frame_path = os.path.join(log_path, last_key_frame_file) 81 82 # Convert lastKeyFrame to numpy array 83 np_image = image_processing_utils.convert_image_to_numpy_array( 84 last_key_frame_path) 85 logging.debug('numpy image shape: %s', np_image.shape) 86 87 return np_image 88 89 90class PreviewVideoZoomMatchTest(its_base_test.ItsBaseTest): 91 """Tests if preview matches video output when zooming. 92 93 Preview and video are recorded while do_3a() iterate through 94 different cameras with minimal zoom to zoom factor 1.5x. 95 96 The recorded preview and video output are processed to dump all 97 of the frames to PNG files. Camera movement in zoom is extracted 98 from frames by determining if the size of the circle being recorded 99 increases as zoom factor increases. Test is a PASS if both recordings 100 match in zoom factors. 101 """ 102 103 def test_preview_video_zoom_match(self): 104 video_test_data = {} 105 preview_test_data = {} 106 log_path = self.log_path 107 with its_session_utils.ItsSession( 108 device_id=self.dut.serial, 109 camera_id=self.camera_id, 110 hidden_physical_id=self.hidden_physical_id) as cam: 111 props = cam.get_camera_properties() 112 props = cam.override_with_hidden_physical_camera_props(props) 113 debug = self.debug_mode 114 115 def _do_preview_recording(cam, resolution, zoom_ratio): 116 """Record a new set of data from the device. 117 118 Captures camera preview frames while the camera is zooming. 119 120 Args: 121 cam: camera object 122 resolution: str; preview resolution (ex. '1920x1080') 123 zoom_ratio: float; zoom ratio 124 125 Returns: 126 preview recording object as described by cam.do_basic_recording 127 """ 128 129 # Record previews 130 preview_recording_obj = cam.do_preview_recording( 131 resolution, _RECORDING_DURATION, False, zoom_ratio=zoom_ratio) 132 logging.debug('Preview_recording_obj: %s', preview_recording_obj) 133 logging.debug('Recorded output path for preview: %s', 134 preview_recording_obj['recordedOutputPath']) 135 136 # Grab and rename the preview recordings from the save location on DUT 137 self.dut.adb.pull( 138 [preview_recording_obj['recordedOutputPath'], log_path]) 139 preview_file_name = ( 140 preview_recording_obj['recordedOutputPath'].split('/')[-1]) 141 logging.debug('recorded preview name: %s', preview_file_name) 142 143 return preview_file_name 144 145 def _do_video_recording(cam, profile_id, quality, zoom_ratio): 146 """Record a new set of data from the device. 147 148 Captures camera video frames while the camera is zooming per zoom_ratio. 149 150 Args: 151 cam: camera object 152 profile_id: int; profile id corresponding to the quality level 153 quality: str; video recording quality such as High, Low, 480P 154 zoom_ratio: float; zoom ratio. 155 156 Returns: 157 video recording object as described by cam.do_basic_recording 158 """ 159 160 # Record videos 161 video_recording_obj = cam.do_basic_recording( 162 profile_id, quality, _RECORDING_DURATION, 0, zoom_ratio=zoom_ratio) 163 logging.debug('Video_recording_obj: %s', video_recording_obj) 164 logging.debug('Recorded output path for video: %s', 165 video_recording_obj['recordedOutputPath']) 166 167 # Grab and rename the video recordings from the save location on DUT 168 self.dut.adb.pull( 169 [video_recording_obj['recordedOutputPath'], log_path]) 170 video_file_name = ( 171 video_recording_obj['recordedOutputPath'].split('/')[-1]) 172 logging.debug('recorded video name: %s', video_file_name) 173 174 return video_file_name 175 176 # Find zoom range 177 z_range = props['android.control.zoomRatioRange'] 178 179 # Skip unless camera has zoom ability 180 vendor_api_level = its_session_utils.get_vendor_api_level( 181 self.dut.serial) 182 camera_properties_utils.skip_unless( 183 z_range and vendor_api_level >= its_session_utils.ANDROID14_API_LEVEL 184 ) 185 logging.debug('Testing zoomRatioRange: %s', str(z_range)) 186 187 # Determine zoom factors 188 z_min = z_range[0] 189 camera_properties_utils.skip_unless( 190 float(z_range[-1]) >= z_min * _ZOOM_MIN_THRESH) 191 zoom_ratios_to_be_tested = [z_min] 192 if z_min < 1.0: 193 zoom_ratios_to_be_tested.append(float(_ZOOM_RATIO)) 194 else: 195 zoom_ratios_to_be_tested.append(float(z_min * 2)) 196 logging.debug('Testing zoom ratios: %s', str(zoom_ratios_to_be_tested)) 197 198 # Load chart for scene 199 if z_min > _MIN_ZOOM_CHART_SCALING: 200 its_session_utils.load_scene( 201 cam, props, self.scene, self.tablet, self.chart_distance) 202 else: 203 its_session_utils.load_scene( 204 cam, props, self.scene, self.tablet, 205 its_session_utils.CHART_DISTANCE_NO_SCALING) 206 207 # Find supported preview/video sizes, and their smallest and common size 208 supported_preview_sizes = cam.get_supported_preview_sizes(self.camera_id) 209 logging.debug('supported_preview_sizes: %s', supported_preview_sizes) 210 supported_video_qualities = cam.get_supported_video_qualities( 211 self.camera_id) 212 logging.debug( 213 'Supported video profiles and ID: %s', supported_video_qualities) 214 common_size, common_video_quality = ( 215 video_processing_utils.get_lowest_preview_video_size( 216 supported_preview_sizes, supported_video_qualities, _MIN_SIZE)) 217 218 # Start video recording over minZoom and 2x Zoom 219 for quality_profile_id_pair in supported_video_qualities: 220 quality = quality_profile_id_pair.split(':')[0] 221 profile_id = quality_profile_id_pair.split(':')[-1] 222 if quality == common_video_quality: 223 for i, z in enumerate(zoom_ratios_to_be_tested): 224 logging.debug('Testing video recording for quality: %s', quality) 225 req = capture_request_utils.auto_capture_request() 226 req['android.control.zoomRatio'] = z 227 cam.do_3a(zoom_ratio=z) 228 logging.debug('Zoom ratio: %.2f', z) 229 230 # Determine focal length of camera through capture 231 cap = cam.do_capture( 232 req, {'format': 'yuv'}) 233 cap_fl = cap['metadata']['android.lens.focalLength'] 234 logging.debug('Camera focal length: %.2f', cap_fl) 235 236 # Determine width and height of video 237 size = common_size.split('x') 238 width = int(size[0]) 239 height = int(size[1]) 240 241 # Start video recording 242 video_file_name = _do_video_recording( 243 cam, profile_id, quality, zoom_ratio=z) 244 245 # Get key frames from the video recording 246 video_img = _extract_key_frame_from_recording( 247 log_path, video_file_name) 248 249 # Find the center circle in video img 250 video_img_name = (f'Video_zoomRatio_{z}_{quality}_circle.png') 251 circle = opencv_processing_utils.find_center_circle( 252 video_img, video_img_name, _CIRCLE_COLOR, 253 circle_ar_rtol=_CIRCLE_AR_RTOL, circlish_rtol=_CIRCLISH_RTOL, 254 min_area=_MIN_AREA_RATIO * width * height * z * z, 255 min_circle_pts=_MIN_CIRCLE_PTS, debug=debug) 256 logging.debug('Recorded video name: %s', video_file_name) 257 258 video_test_data[i] = {'z': z, 'circle': circle} 259 260 # Start preview recording over minZoom and maxZoom 261 for size in supported_preview_sizes: 262 if size == common_size: 263 for i, z in enumerate(zoom_ratios_to_be_tested): 264 cam.do_3a(zoom_ratio=z) 265 preview_file_name = _do_preview_recording( 266 cam, size, zoom_ratio=z) 267 268 # Define width and height from size 269 width = int(size.split('x')[0]) 270 height = int(size.split('x')[1]) 271 272 # Get key frames from the preview recording 273 preview_img = _extract_key_frame_from_recording( 274 log_path, preview_file_name) 275 276 # If testing front camera, mirror preview image 277 # Opencv expects a numpy array but np.flip generates a 'view' which 278 # doesn't work with opencv. ndarray.copy forces copy instead of view 279 if props['android.lens.facing'] == _LENS_FACING_FRONT: 280 # Preview are flipped on device's natural orientation 281 # so for sensor orientation 90 or 270, it is up or down 282 # Sensor orientation 0 or 180 is left or right 283 if props['android.sensor.orientation'] in (90, 270): 284 preview_img = np.ndarray.copy(np.flipud(preview_img)) 285 logging.debug( 286 'Found sensor orientation %d, flipping up down', 287 props['android.sensor.orientation']) 288 else: 289 preview_img = np.ndarray.copy(np.fliplr(preview_img)) 290 logging.debug( 291 'Found sensor orientation %d, flipping left right', 292 props['android.sensor.orientation']) 293 294 # Find the center circle in preview img 295 preview_img_name = (f'Preview_zoomRatio_{z}_{size}_circle.png') 296 circle = opencv_processing_utils.find_center_circle( 297 preview_img, preview_img_name, _CIRCLE_COLOR, 298 circle_ar_rtol=_CIRCLE_AR_RTOL, circlish_rtol=_CIRCLISH_RTOL, 299 min_area=_MIN_AREA_RATIO * width * height * z * z, 300 min_circle_pts=_MIN_CIRCLE_PTS, debug=debug) 301 if opencv_processing_utils.is_circle_cropped( 302 circle, (width, height)): 303 logging.debug('Zoom %.2f is too large!', z) 304 305 preview_test_data[i] = {'z': z, 'circle': circle} 306 307 # compare size and center of preview's circle to video's circle 308 preview_radius = {} 309 video_radius = {} 310 z_idx = {} 311 zoom_factor = {} 312 preview_radius[_MIN_STR] = (preview_test_data[0]['circle'][_CIRCLE_R]) 313 video_radius[_MIN_STR] = (video_test_data[0]['circle'][_CIRCLE_R]) 314 preview_radius[_MAX_STR] = (preview_test_data[1]['circle'][_CIRCLE_R]) 315 video_radius[_MAX_STR] = (video_test_data[1]['circle'][_CIRCLE_R]) 316 z_idx[_MIN_STR] = ( 317 preview_radius[_MIN_STR] / video_radius[_MIN_STR]) 318 z_idx[_MAX_STR] = ( 319 preview_radius[_MAX_STR] / video_radius[_MAX_STR]) 320 z_comparison = z_idx[_MAX_STR] / z_idx[_MIN_STR] 321 zoom_factor[_MIN_STR] = preview_test_data[0]['z'] 322 zoom_factor[_MAX_STR] = preview_test_data[1]['z'] 323 324 # compare preview circle's center with video circle's center 325 preview_circle_x = preview_test_data[1]['circle'][_CIRCLE_X] 326 video_circle_x = video_test_data[1]['circle'][_CIRCLE_X] 327 preview_circle_y = preview_test_data[1]['circle'][_CIRCLE_Y] 328 video_circle_y = video_test_data[1]['circle'][_CIRCLE_Y] 329 circles_offset_x = math.isclose(preview_circle_x, video_circle_x, 330 abs_tol=_OFFSET_TOL) 331 circles_offset_y = math.isclose(preview_circle_y, video_circle_y, 332 abs_tol=_OFFSET_TOL) 333 logging.debug('Preview circle x: %.2f, Video circle x: %.2f' 334 ' Preview circle y: %.2f, Video circle y: %.2f', 335 preview_circle_x, video_circle_x, 336 preview_circle_y, video_circle_y) 337 logging.debug('Preview circle r: %.2f, Preview circle r zoom: %.2f' 338 ' Video circle r: %.2f, Video circle r zoom: %.2f' 339 ' centers offset x: %s, centers offset y: %s', 340 preview_radius[_MIN_STR], preview_radius[_MAX_STR], 341 video_radius[_MIN_STR], video_radius[_MAX_STR], 342 circles_offset_x, circles_offset_y) 343 if not circles_offset_x or not circles_offset_y: 344 raise AssertionError('Preview and video output do not match!' 345 ' Preview and video circles offset is too great') 346 347 # check zoom ratio by size of circles before and after zoom 348 for radius_ratio in z_idx.values(): 349 if not math.isclose(radius_ratio, 1, rel_tol=_RADIUS_RTOL): 350 raise AssertionError('Preview and video output do not match!' 351 ' Radius ratio: %.2f', radius_ratio) 352 353 if z_comparison > _ZOOM_COMP_MAX_THRESH: 354 raise AssertionError('Preview and video output do not match!' 355 ' Zoom ratio difference: %.2f', z_comparison) 356 357if __name__ == '__main__': 358 test_runner.main() 359 360