1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Verify preview is stable during phone movement.""" 15 16import logging 17import os 18import threading 19import time 20 21from mobly import test_runner 22 23import its_base_test 24import camera_properties_utils 25import image_processing_utils 26import its_session_utils 27import opencv_processing_utils 28import sensor_fusion_utils 29import video_processing_utils 30 31_ARDUINO_ANGLES = (10, 25) # degrees 32_ARDUINO_MOVE_TIME = 0.30 # seconds 33_ARDUINO_SERVO_SPEED = 10 34_ASPECT_RATIO_16_9 = 16/9 # determine if preview fmt > 16:9 35_IMG_FORMAT = 'png' 36_MIN_PHONE_MOVEMENT_ANGLE = 5 # degrees 37_NAME = os.path.splitext(os.path.basename(__file__))[0] 38_NUM_ROTATIONS = 24 39_START_FRAME = 30 # give 3A some frames to warm up 40_TABLET_SERVO_SPEED = 20 41_TEST_REQUIRED_MPC = 33 42_VIDEO_DELAY_TIME = 5.5 # seconds 43_VIDEO_DURATION = 5.5 # seconds 44_PREVIEW_STABILIZATION_FACTOR = 0.7 # 70% of gyro movement allowed 45_PREVIEW_STABILIZATION_MODE_PREVIEW = 2 46 47 48def _collect_data(cam, tablet_device, video_size, rot_rig): 49 """Capture a new set of data from the device. 50 51 Captures camera preview frames while the user is moving the device in 52 the prescribed manner. 53 54 Args: 55 cam: camera object 56 tablet_device: boolean; based on config file 57 video_size: str; video resolution. ex. '1920x1080' 58 rot_rig: dict with 'cntl' and 'ch' defined 59 60 Returns: 61 recording object as described by cam.do_preview_recording 62 """ 63 64 logging.debug('Starting sensor event collection') 65 serial_port = None 66 if rot_rig['cntl'].lower() == sensor_fusion_utils.ARDUINO_STRING.lower(): 67 # identify port 68 serial_port = sensor_fusion_utils.serial_port_def( 69 sensor_fusion_utils.ARDUINO_STRING) 70 # send test cmd to Arduino until cmd returns properly 71 sensor_fusion_utils.establish_serial_comm(serial_port) 72 # Start camera vibration 73 if tablet_device: 74 servo_speed = _TABLET_SERVO_SPEED 75 else: 76 servo_speed = _ARDUINO_SERVO_SPEED 77 p = threading.Thread( 78 target=sensor_fusion_utils.rotation_rig, 79 args=( 80 rot_rig['cntl'], 81 rot_rig['ch'], 82 _NUM_ROTATIONS, 83 _ARDUINO_ANGLES, 84 servo_speed, 85 _ARDUINO_MOVE_TIME, 86 serial_port, 87 ), 88 ) 89 p.start() 90 91 cam.start_sensor_events() 92 # Record video and return recording object 93 time.sleep(_VIDEO_DELAY_TIME) # allow time for rig to start moving 94 95 recording_obj = cam.do_preview_recording(video_size, _VIDEO_DURATION, True) 96 logging.debug('Recorded output path: %s', recording_obj['recordedOutputPath']) 97 logging.debug('Tested quality: %s', recording_obj['quality']) 98 99 # Wait for vibration to stop 100 p.join() 101 102 return recording_obj 103 104 105class PreviewStabilizationTest(its_base_test.ItsBaseTest): 106 """Tests if preview is stabilized. 107 108 Camera is moved in sensor fusion rig on an arc of 15 degrees. 109 Speed is set to mimic hand movement (and not be too fast). 110 Preview is captured after rotation rig starts moving, and the 111 gyroscope data is dumped. 112 113 The recorded preview is processed to dump all of the frames to 114 PNG files. Camera movement is extracted from frames by determining 115 max angle of deflection in video movement vs max angle of deflection 116 in gyroscope movement. Test is a PASS if rotation is reduced in video. 117 """ 118 119 def test_preview_stabilization(self): 120 rot_rig = {} 121 log_path = self.log_path 122 123 with its_session_utils.ItsSession( 124 device_id=self.dut.serial, 125 camera_id=self.camera_id, 126 hidden_physical_id=self.hidden_physical_id) as cam: 127 128 props = cam.get_camera_properties() 129 props = cam.override_with_hidden_physical_camera_props(props) 130 first_api_level = its_session_utils.get_first_api_level(self.dut.serial) 131 camera_properties_utils.skip_unless( 132 first_api_level >= its_session_utils.ANDROID13_API_LEVEL, 133 'First API level should be {} or higher. Found {}.'.format( 134 its_session_utils.ANDROID13_API_LEVEL, first_api_level)) 135 136 supported_stabilization_modes = props[ 137 'android.control.availableVideoStabilizationModes' 138 ] 139 140 # Check media performance class 141 should_run = (supported_stabilization_modes is not None and 142 _PREVIEW_STABILIZATION_MODE_PREVIEW in 143 supported_stabilization_modes) 144 media_performance_class = its_session_utils.get_media_performance_class( 145 self.dut.serial) 146 if media_performance_class >= _TEST_REQUIRED_MPC and not should_run: 147 its_session_utils.raise_mpc_assertion_error( 148 _TEST_REQUIRED_MPC, _NAME, media_performance_class) 149 150 camera_properties_utils.skip_unless(should_run) 151 152 # Calculate camera FoV and convert from string to float 153 camera_fov = float(cam.calc_camera_fov(props)) 154 155 # Log ffmpeg version being used 156 video_processing_utils.log_ffmpeg_version() 157 158 # Raise error if not FRONT or REAR facing camera 159 facing = props['android.lens.facing'] 160 if (facing != camera_properties_utils.LENS_FACING_BACK 161 and facing != camera_properties_utils.LENS_FACING_FRONT): 162 raise AssertionError('Unknown lens facing: {facing}.') 163 164 # Initialize rotation rig 165 rot_rig['cntl'] = self.rotator_cntl 166 rot_rig['ch'] = self.rotator_ch 167 if rot_rig['cntl'].lower() != 'arduino': 168 raise AssertionError( 169 f'You must use the arduino controller for {_NAME}.') 170 171 # List of video resolutions to test 172 lowest_res_tested = video_processing_utils.LOWEST_RES_TESTED_AREA 173 resolution_to_area = lambda s: int(s.split('x')[0])*int(s.split('x')[1]) 174 supported_preview_sizes = cam.get_supported_preview_sizes(self.camera_id) 175 supported_preview_sizes = [size for size in supported_preview_sizes 176 if resolution_to_area(size) 177 >= lowest_res_tested] 178 logging.debug('Supported preview resolutions: %s', 179 supported_preview_sizes) 180 181 max_cam_gyro_angles = {} 182 183 for video_size in supported_preview_sizes: 184 recording_obj = _collect_data(cam, self.tablet_device, video_size, rot_rig) 185 186 # Grab the video from the save location on DUT 187 self.dut.adb.pull([recording_obj['recordedOutputPath'], log_path]) 188 file_name = recording_obj['recordedOutputPath'].split('/')[-1] 189 logging.debug('recorded file name: %s', file_name) 190 191 # Get gyro events 192 logging.debug('Reading out inertial sensor events') 193 gyro_events = cam.get_sensor_events()['gyro'] 194 logging.debug('Number of gyro samples %d', len(gyro_events)) 195 196 # Get all frames from the video 197 file_list = video_processing_utils.extract_all_frames_from_video( 198 log_path, file_name, _IMG_FORMAT 199 ) 200 frames = [] 201 202 logging.debug('Number of frames %d', len(file_list)) 203 for file in file_list: 204 img = image_processing_utils.convert_image_to_numpy_array( 205 os.path.join(log_path, file) 206 ) 207 frames.append(img / 255) 208 frame_shape = frames[0].shape 209 logging.debug('Frame size %d x %d', frame_shape[1], frame_shape[0]) 210 211 # Extract camera rotations 212 img_h = frames[0].shape[0] 213 file_name_stem = f'{os.path.join(log_path, _NAME)}_{video_size}' 214 cam_rots = sensor_fusion_utils.get_cam_rotations( 215 frames[_START_FRAME : len(frames)], 216 facing, 217 img_h, 218 file_name_stem, 219 _START_FRAME, 220 stabilized_video=True 221 ) 222 sensor_fusion_utils.plot_camera_rotations(cam_rots, _START_FRAME, 223 video_size, file_name_stem) 224 max_camera_angle = sensor_fusion_utils.calc_max_rotation_angle( 225 cam_rots, 'Camera') 226 227 # Extract gyro rotations 228 sensor_fusion_utils.plot_gyro_events( 229 gyro_events, f'{_NAME}_{video_size}', log_path 230 ) 231 gyro_rots = sensor_fusion_utils.conv_acceleration_to_movement( 232 gyro_events, _VIDEO_DELAY_TIME 233 ) 234 max_gyro_angle = sensor_fusion_utils.calc_max_rotation_angle( 235 gyro_rots, 'Gyro') 236 logging.debug( 237 'Max deflection (degrees) %s: video: %.3f, gyro: %.3f ratio: %.4f', 238 video_size, max_camera_angle, max_gyro_angle, 239 max_camera_angle / max_gyro_angle) 240 max_cam_gyro_angles[video_size] = {'gyro': max_gyro_angle, 241 'cam': max_camera_angle} 242 243 # Assert phone is moved enough during test 244 if max_gyro_angle < _MIN_PHONE_MOVEMENT_ANGLE: 245 raise AssertionError( 246 f'Phone not moved enough! Movement: {max_gyro_angle}, ' 247 f'THRESH: {_MIN_PHONE_MOVEMENT_ANGLE} degrees') 248 249 # Assert PASS/FAIL criteria 250 test_failures = [] 251 for preview_size, max_angles in max_cam_gyro_angles.items(): 252 w_x_h = preview_size.split('x') 253 if int(w_x_h[0])/int(w_x_h[1]) > _ASPECT_RATIO_16_9: 254 preview_stabilization_factor = _PREVIEW_STABILIZATION_FACTOR * 1.1 255 else: 256 preview_stabilization_factor = _PREVIEW_STABILIZATION_FACTOR 257 if max_angles['cam'] >= max_angles['gyro']*preview_stabilization_factor: 258 test_failures.append( 259 f'{preview_size} preview not stabilized enough! ' 260 f"Max preview angle: {max_angles['cam']:.3f}, " 261 f"Max gyro angle: {max_angles['gyro']:.3f}, " 262 f"ratio: {max_angles['cam']/max_angles['gyro']:.3f} " 263 f'THRESH: {preview_stabilization_factor}.') 264 265 if test_failures: 266 raise AssertionError(test_failures) 267 268 269if __name__ == '__main__': 270 test_runner.main() 271 272