1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Verify video is stable during phone movement.""" 15 16import logging 17import math 18import os 19import threading 20import time 21 22from mobly import test_runner 23 24import its_base_test 25import camera_properties_utils 26import image_processing_utils 27import its_session_utils 28import opencv_processing_utils 29import sensor_fusion_utils 30import video_processing_utils 31 32_ARDUINO_ANGLES = (10, 25) # degrees 33_ARDUINO_MOVE_TIME = 0.30 # seconds 34_ARDUINO_SERVO_SPEED = 10 35_ASPECT_RATIO_16_9 = 16/9 # determine if video fmt > 16:9 36_IMG_FORMAT = 'png' 37_MIN_PHONE_MOVEMENT_ANGLE = 5 # degrees 38_NAME = os.path.splitext(os.path.basename(__file__))[0] 39_NUM_ROTATIONS = 24 40_RADS_TO_DEGS = 180/math.pi 41_SEC_TO_NSEC = 1E9 42_START_FRAME = 30 # give 3A 1s to warm up 43_TABLET_SERVO_SPEED = 20 44_VIDEO_DELAY_TIME = 5.5 # seconds 45_VIDEO_DURATION = 5.5 # seconds 46_VIDEO_QUALITIES_TESTED = ('CIF:3', '480P:4', '720P:5', '1080P:6', 'QVGA:7', 47 'VGA:9') 48_VIDEO_STABILIZATION_FACTOR = 0.7 # 70% of gyro movement allowed 49_VIDEO_STABILIZATION_MODE = 1 50_SIZE_TO_PROFILE = {'176x144': 'QCIF:2', '352x288': 'CIF:3', 51 '320x240': 'QVGA:7'} 52 53 54def _collect_data(cam, tablet_device, video_profile, video_quality, rot_rig): 55 """Capture a new set of data from the device. 56 57 Captures camera frames while the user is moving the device in the prescribed 58 manner. 59 60 Args: 61 cam: camera object 62 tablet_device: boolean; based on config.yml 63 video_profile: str; number of video profile 64 video_quality: str; key string for video quality. ie. 1080P 65 rot_rig: dict with 'cntl' and 'ch' defined 66 67 Returns: 68 recording object 69 """ 70 logging.debug('Starting sensor event collection') 71 props = cam.get_camera_properties() 72 props = cam.override_with_hidden_physical_camera_props(props) 73 74 serial_port = None 75 if rot_rig['cntl'].lower() == sensor_fusion_utils.ARDUINO_STRING.lower(): 76 # identify port 77 serial_port = sensor_fusion_utils.serial_port_def( 78 sensor_fusion_utils.ARDUINO_STRING) 79 # send test cmd to Arduino until cmd returns properly 80 sensor_fusion_utils.establish_serial_comm(serial_port) 81 # Start camera vibration 82 if tablet_device: 83 servo_speed = _TABLET_SERVO_SPEED 84 else: 85 servo_speed = _ARDUINO_SERVO_SPEED 86 87 p = threading.Thread( 88 target=sensor_fusion_utils.rotation_rig, 89 args=(rot_rig['cntl'], rot_rig['ch'], _NUM_ROTATIONS, 90 _ARDUINO_ANGLES, servo_speed, _ARDUINO_MOVE_TIME, serial_port)) 91 p.start() 92 93 cam.start_sensor_events() 94 95 # Record video and return recording object 96 time.sleep(_VIDEO_DELAY_TIME) # allow time for rig to start moving 97 recording_obj = cam.do_basic_recording( 98 video_profile, video_quality, _VIDEO_DURATION, _VIDEO_STABILIZATION_MODE) 99 logging.debug('Recorded output path: %s', recording_obj['recordedOutputPath']) 100 logging.debug('Tested quality: %s', recording_obj['quality']) 101 102 # Wait for vibration to stop 103 p.join() 104 105 return recording_obj 106 107 108class VideoStabilizationTest(its_base_test.ItsBaseTest): 109 """Tests if video is stabilized. 110 111 Camera is moved in sensor fusion rig on an arc of 15 degrees. 112 Speed is set to mimic hand movement (and not be too fast.) 113 Video is captured after rotation rig starts moving, and the 114 gyroscope data is dumped. 115 116 Video is processed to dump all of the frames to PNG files. 117 Camera movement is extracted from frames by determining max 118 angle of deflection in video movement vs max angle of deflection 119 in gyroscope movement. Test is a PASS if rotation is reduced in video. 120 """ 121 122 def test_video_stabilization(self): 123 rot_rig = {} 124 log_path = self.log_path 125 126 with its_session_utils.ItsSession( 127 device_id=self.dut.serial, 128 camera_id=self.camera_id, 129 hidden_physical_id=self.hidden_physical_id) as cam: 130 props = cam.get_camera_properties() 131 props = cam.override_with_hidden_physical_camera_props(props) 132 vendor_api_level = its_session_utils.get_vendor_api_level(self.dut.serial) 133 supported_stabilization_modes = props[ 134 'android.control.availableVideoStabilizationModes'] 135 136 camera_properties_utils.skip_unless( 137 vendor_api_level >= its_session_utils.ANDROID13_API_LEVEL and 138 _VIDEO_STABILIZATION_MODE in supported_stabilization_modes) 139 140 # Calculate camera FoV and convert from string to float 141 camera_fov = float(cam.calc_camera_fov(props)) 142 143 # Log ffmpeg version being used 144 video_processing_utils.log_ffmpeg_version() 145 146 # Raise error if not FRONT or REAR facing camera 147 facing = props['android.lens.facing'] 148 if (facing != camera_properties_utils.LENS_FACING_FRONT and 149 facing != camera_properties_utils.LENS_FACING_BACK): 150 raise AssertionError(f'Unknown lens facing: {facing}.') 151 152 # Initialize rotation rig 153 rot_rig['cntl'] = self.rotator_cntl 154 rot_rig['ch'] = self.rotator_ch 155 if rot_rig['cntl'].lower() != 'arduino': 156 raise AssertionError(f'You must use an arduino controller for {_NAME}.') 157 158 # Create list of video qualities to test 159 excluded_sizes = video_processing_utils.LOW_RESOLUTION_SIZES 160 excluded_qualities = [ 161 _SIZE_TO_PROFILE[s] for s in excluded_sizes if s in _SIZE_TO_PROFILE 162 ] 163 supported_video_qualities = cam.get_supported_video_qualities( 164 self.camera_id) 165 logging.debug('Supported video qualities: %s', supported_video_qualities) 166 tested_video_qualities = list(set(_VIDEO_QUALITIES_TESTED) & 167 set(supported_video_qualities) - 168 set(excluded_qualities)) 169 170 # Raise error if no video qualities to test 171 if not tested_video_qualities: 172 raise AssertionError( 173 f'QUALITY_LOW not supported: {supported_video_qualities}') 174 else: 175 logging.debug('video qualities tested: %s', str(tested_video_qualities)) 176 177 max_cam_gyro_angles = {} 178 179 for video_tested in tested_video_qualities: 180 video_profile = video_tested.split(':')[1] 181 video_quality = video_tested.split(':')[0] 182 183 # Record video 184 recording_obj = _collect_data( 185 cam, self.tablet_device, video_profile, video_quality, rot_rig) 186 187 # Grab the video from the save location on DUT 188 self.dut.adb.pull([recording_obj['recordedOutputPath'], log_path]) 189 file_name = recording_obj['recordedOutputPath'].split('/')[-1] 190 logging.debug('file_name: %s', file_name) 191 192 # Get gyro events 193 logging.debug('Reading out inertial sensor events') 194 gyro_events = cam.get_sensor_events()['gyro'] 195 logging.debug('Number of gyro samples %d', len(gyro_events)) 196 197 # Extract all frames from video 198 file_list = video_processing_utils.extract_all_frames_from_video( 199 log_path, file_name, _IMG_FORMAT) 200 frames = [] 201 logging.debug('Number of frames %d', len(file_list)) 202 for file in file_list: 203 img = image_processing_utils.convert_image_to_numpy_array( 204 os.path.join(log_path, file)) 205 frames.append(img/255) 206 frame_shape = frames[0].shape 207 logging.debug('Frame size %d x %d', frame_shape[1], frame_shape[0]) 208 209 # Extract camera rotations 210 img_h = frames[0].shape[0] 211 file_name_stem = f'{os.path.join(log_path, _NAME)}_{video_quality}' 212 cam_rots = sensor_fusion_utils.get_cam_rotations( 213 frames[_START_FRAME:len(frames)], facing, img_h, 214 file_name_stem, _START_FRAME, stabilized_video=True) 215 sensor_fusion_utils.plot_camera_rotations( 216 cam_rots, _START_FRAME, video_quality, file_name_stem) 217 max_camera_angle = sensor_fusion_utils.calc_max_rotation_angle( 218 cam_rots, 'Camera') 219 220 # Extract gyro rotations 221 sensor_fusion_utils.plot_gyro_events( 222 gyro_events, f'{_NAME}_{video_quality}', log_path) 223 gyro_rots = sensor_fusion_utils.conv_acceleration_to_movement( 224 gyro_events, _VIDEO_DELAY_TIME) 225 max_gyro_angle = sensor_fusion_utils.calc_max_rotation_angle( 226 gyro_rots, 'Gyro') 227 logging.debug( 228 'Max deflection (degrees) %s: video: %.3f, gyro: %.3f, ratio: %.4f', 229 video_quality, max_camera_angle, max_gyro_angle, 230 max_camera_angle / max_gyro_angle) 231 max_cam_gyro_angles[video_quality] = {'gyro': max_gyro_angle, 232 'cam': max_camera_angle, 233 'frame_shape': frame_shape} 234 235 # Assert phone is moved enough during test 236 if max_gyro_angle < _MIN_PHONE_MOVEMENT_ANGLE: 237 raise AssertionError( 238 f'Phone not moved enough! Movement: {max_gyro_angle}, ' 239 f'THRESH: {_MIN_PHONE_MOVEMENT_ANGLE} degrees') 240 241 # Assert PASS/FAIL criteria 242 test_failures = [] 243 for video_quality, max_angles in max_cam_gyro_angles.items(): 244 aspect_ratio = (max_angles['frame_shape'][1] / 245 max_angles['frame_shape'][0]) 246 if aspect_ratio > _ASPECT_RATIO_16_9: 247 video_stabilization_factor = _VIDEO_STABILIZATION_FACTOR * 1.1 248 else: 249 video_stabilization_factor = _VIDEO_STABILIZATION_FACTOR 250 if max_angles['cam'] >= max_angles['gyro']*video_stabilization_factor: 251 test_failures.append( 252 f'{video_quality} video not stabilized enough! ' 253 f"Max video angle: {max_angles['cam']:.3f}, " 254 f"Max gyro angle: {max_angles['gyro']:.3f}, " 255 f"ratio: {max_angles['cam']/max_angles['gyro']:.3f} " 256 f'THRESH: {video_stabilization_factor}.') 257 if test_failures: 258 raise AssertionError(test_failures) 259 260 261if __name__ == '__main__': 262 test_runner.main() 263