• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2024 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verify that lens intrinsics changes when OIS is triggered."""
15
16import logging
17import math
18import numpy as np
19import os
20
21from matplotlib import pyplot as plt
22from mobly import test_runner
23
24import its_base_test
25import camera_properties_utils
26import its_session_utils
27import preview_processing_utils
28import sensor_fusion_utils
29
30_INTRINSICS_SAMPLES = 'android.statistics.lensIntrinsicsSamples'
31_NAME = os.path.splitext(os.path.basename(__file__))[0]
32_MIN_PHONE_MOVEMENT_ANGLE = 5  # degrees
33_PRINCIPAL_POINT_THRESH = 1  # Threshold for principal point changes in pixels.
34_START_FRAME = 30  # give 3A some frames to warm up
35_VIDEO_DELAY_TIME = 5.5  # seconds
36
37# Note: b/284232490: 1080p could be 1088. 480p could be 704 or 640 too.
38#       Use for tests not sensitive to variations of 1080p or 480p.
39# TODO: b/370841141 - Remove usage of VIDEO_PREVIEW_QUALITY_SIZE.
40#                     Create and use get_supported_video_sizes instead of
41#                     get_supported_video_qualities.
42_VIDEO_PREVIEW_QUALITY_SIZE = {
43    # 'HIGH' and 'LOW' not included as they are DUT-dependent
44    '4KDC': '4096x2160',
45    '2160P': '3840x2160',
46    'QHD': '2560x1440',
47    '2k': '2048x1080',
48    '1080P': '1920x1080',
49    '720P': '1280x720',
50    '480P': '720x480',
51    'VGA': '640x480',
52    'CIF': '352x288',
53    'QVGA': '320x240',
54    'QCIF': '176x144',
55}
56
57
58def get_largest_video_size(cam, camera_id):
59  """Returns the largest supported video size and its area.
60
61  Determine largest supported video size and its area from
62  get_supported_video_qualities.
63
64  Args:
65    cam: camera object.
66    camera_id: str; camera ID.
67
68  Returns:
69    max_size: str; largest supported video size in the format 'widthxheight'.
70    max_area: int; area of the largest supported video size.
71  """
72  supported_video_qualities = cam.get_supported_video_qualities(camera_id)
73  logging.debug('Supported video profiles & IDs: %s',
74                supported_video_qualities)
75
76  quality_keys = [
77      quality.split(':')[0]
78      for quality in supported_video_qualities
79  ]
80  logging.debug('Quality keys: %s', quality_keys)
81
82  supported_video_sizes = [
83      _VIDEO_PREVIEW_QUALITY_SIZE[key]
84      for key in quality_keys
85      if key in _VIDEO_PREVIEW_QUALITY_SIZE
86  ]
87  logging.debug('Supported video sizes: %s', supported_video_sizes)
88
89  if not supported_video_sizes:
90    raise AssertionError('No supported video sizes found!')
91
92  size_to_area = lambda s: int(s.split('x')[0])*int(s.split('x')[1])
93  max_size = max(supported_video_sizes, key=size_to_area)
94
95  logging.debug('Largest video size: %s', max_size)
96  return size_to_area(max_size)
97
98
99def calculate_principal_point(f_x, f_y, c_x, c_y, s):
100  """Calculates the principal point of a camera given its intrinsic parameters.
101
102  Args:
103    f_x: Horizontal focal length.
104    f_y: Vertical focal length.
105    c_x: X coordinate of the optical axis.
106    c_y: Y coordinate of the optical axis.
107    s: Skew parameter.
108
109  Returns:
110    A numpy array containing the principal point coordinates (px, py).
111  """
112
113  # Create the camera calibration matrix
114  transform_k = np.array([[f_x, s, c_x],
115                          [0, f_y, c_y],
116                          [0, 0, 1]])
117
118  # The Principal point is the intersection of the optical axis with the
119  # image plane. Since the optical axis passes through the camera center
120  # (defined by K), the principal point coordinates are simply the
121  # projection of the camera center onto the image plane.
122  principal_point = np.dot(transform_k, np.array([0, 0, 1]))
123
124  # Normalize by the homogeneous coordinate
125  px = principal_point[0] / principal_point[2]
126  py = principal_point[1] / principal_point[2]
127
128  return px, py
129
130
131def plot_principal_points(principal_points_dist, start_frame,
132                          video_quality, plot_name_stem):
133  """Plot principal points values vs Camera frames.
134
135  Args:
136    principal_points_dist: array of principal point distances in pixels/frame
137    start_frame: int value of start frame
138    video_quality: str for video quality identifier
139    plot_name_stem: str; name of the plot
140  """
141
142  plt.figure(video_quality)
143  frames = range(start_frame, len(principal_points_dist)+start_frame)
144  plt.title(f'Lens Intrinsics vs frame {video_quality}')
145  plt.plot(frames, principal_points_dist, '-ro', label='dist')
146  plt.xlabel('Frame #')
147  plt.ylabel('Principal points in pixels')
148  plt.savefig(f'{plot_name_stem}.png')
149  plt.close(video_quality)
150
151
152def verify_lens_intrinsics(recording_obj, gyro_events, test_name, log_path):
153  """Verify principal points changes due to OIS changes.
154
155  Args:
156    recording_obj: Camcorder recording object.
157    gyro_events: Gyroscope events collected while recording.
158    test_name: Name of the test
159    log_path: Path for the log file
160
161  Returns:
162    A dictionary containing the maximum gyro angle, the maximum changes of
163    principal point, and a failure message if principal point doesn't change
164    due to OIS changes triggered by device motion.
165  """
166
167  file_name = recording_obj['recordedOutputPath'].split('/')[-1]
168  logging.debug('recorded file name: %s', file_name)
169  video_size = recording_obj['videoSize']
170  logging.debug('video size: %s', video_size)
171
172  capture_results = recording_obj['captureMetadata']
173  file_name_stem = f'{os.path.join(log_path, test_name)}_{video_size}'
174
175  # Extract principal points from capture result
176  principal_points = []
177  for capture_result in capture_results:
178    if capture_result.get('android.lens.intrinsicCalibration'):
179      intrinsic_cal = capture_result['android.lens.intrinsicCalibration']
180      logging.debug('Intrinsic Calibration: %s', str(intrinsic_cal))
181      principal_points.append(calculate_principal_point(*intrinsic_cal[:5]))
182
183  if not principal_points:
184    logging.debug('Lens Intrinsic are not reported in Capture Results.')
185    return {'gyro': None, 'max_pp_diff': None,
186            'failure': None, 'skip': True}
187
188  # Calculate variations in principal points
189  first_point = principal_points[0]
190  principal_points_diff = [math.dist(first_point, x) for x in principal_points]
191
192  plot_principal_points(principal_points_diff,
193                        _START_FRAME,
194                        video_size,
195                        file_name_stem)
196
197  max_pp_diff = max(principal_points_diff)
198
199  # Extract gyro rotations
200  sensor_fusion_utils.plot_gyro_events(
201      gyro_events, f'{test_name}_{video_size}', log_path)
202  gyro_rots = sensor_fusion_utils.conv_acceleration_to_movement(
203      gyro_events, _VIDEO_DELAY_TIME)
204  max_gyro_angle = sensor_fusion_utils.calc_max_rotation_angle(
205      gyro_rots, 'Gyro')
206  logging.debug(
207      'Max deflection (degrees) %s: gyro: %.3f',
208      video_size, max_gyro_angle)
209
210  # Assert phone is moved enough during test
211  if max_gyro_angle < _MIN_PHONE_MOVEMENT_ANGLE:
212    raise AssertionError(
213        f'Phone not moved enough! Movement: {max_gyro_angle}, '
214        f'THRESH: {_MIN_PHONE_MOVEMENT_ANGLE} degrees')
215
216  failure_msg = None
217  if(max_pp_diff > _PRINCIPAL_POINT_THRESH):
218    logging.debug('Principal point diff: x = %.2f', max_pp_diff)
219  else:
220    failure_msg = (
221        'Change in principal point not enough with respect to OIS changes. '
222        f'video_size: {video_size}, '
223        f'Max Principal Point deflection (pixels):  {max_pp_diff:.3f}, '
224        f'Max gyro angle: {max_gyro_angle:.3f}, '
225        f'THRESHOLD : {_PRINCIPAL_POINT_THRESH}.')
226
227  return {'gyro': max_gyro_angle, 'max_pp_diff': max_pp_diff,
228          'failure': failure_msg, 'skip': False}
229
230
231def verify_lens_intrinsics_sample(recording_obj):
232  """Verify principal points changes in intrinsics samples.
233
234  Validate if principal points changes in at least one intrinsics samples.
235  Validate if timestamp changes in each intrinsics samples.
236
237  Args:
238    recording_obj: Camcorder recording object.
239
240  Returns:
241    a failure message if principal point doesn't change.
242    a failure message if timestamps doesn't change
243    None: either test passes or capture results doesn't include
244          intrinsics samples
245  """
246
247  file_name = recording_obj['recordedOutputPath'].split('/')[-1]
248  logging.debug('recorded file name: %s', file_name)
249  video_size = recording_obj['videoSize']
250  logging.debug('video size: %s', video_size)
251
252  capture_results = recording_obj['captureMetadata']
253
254  # Extract Lens Intrinsics Samples from capture result
255  intrinsics_samples_list = []
256  for capture_result in capture_results:
257    if _INTRINSICS_SAMPLES in capture_result:
258      samples = capture_result[_INTRINSICS_SAMPLES]
259      intrinsics_samples_list.append(samples)
260
261  if not intrinsics_samples_list:
262    logging.debug('Lens Intrinsic Samples are not reported')
263    # Don't change print to logging. Used for KPI.
264    print(f'{_NAME}_samples_principal_points_diff_detected: false')
265    return {'failure': None, 'skip': True}
266
267  failure_msg = ''
268
269  max_samples_pp_diffs = []
270  max_samples_timestamp_diffs = []
271  for samples in intrinsics_samples_list:
272    pp_diffs = []
273    timestamp_diffs = []
274
275    # Evaluate intrinsics samples
276    first_sample = samples[0]
277    first_instrinsics = first_sample['lensIntrinsics']
278    first_ts = first_sample['timestamp']
279    first_point = calculate_principal_point(*first_instrinsics[:5])
280
281    for sample in samples:
282      samples_intrinsics = sample['lensIntrinsics']
283      timestamp = sample['timestamp']
284      principal_point = calculate_principal_point(*samples_intrinsics[:5])
285      distance = math.dist(first_point, principal_point)
286      pp_diffs.append(distance)
287      timestamp_diffs.append(timestamp-first_ts)
288
289    max_samples_pp_diffs.append(max(pp_diffs))
290    max_samples_timestamp_diffs.append(max(timestamp_diffs))
291
292  if any(value != 0 for value in max_samples_pp_diffs):
293    # Don't change print to logging. Used for KPI.
294    print(f'{_NAME}_samples_principal_points_diff_detected: true')
295    logging.debug('Principal points variations found in at lease one sample')
296  else:
297    # Don't change print to logging. Used for KPI.
298    print(f'{_NAME}_samples_principal_points_diff_detected: false')
299    failure_msg = failure_msg + (
300        'No variation of principal points found in any samples.\n\n'
301    )
302  if all(diff > 0 for diff in max_samples_timestamp_diffs[1:]):
303    logging.debug('Timestamps variations found in all samples')
304  else:
305    failure_msg = failure_msg + 'Timestamps in samples did not change. \n\n'
306
307  failure_msg = None if failure_msg else failure_msg
308
309  return {'failure': failure_msg, 'skip': False}
310
311
312class LensIntrinsicCalibrationTest(its_base_test.ItsBaseTest):
313  """Tests if lens intrinsics changes when OIS is triggered.
314
315  Camera is moved in sensor fusion rig on an angle of 15 degrees.
316  Speed is set to mimic hand movement (and not be too fast).
317  Preview is recorded after rotation rig starts moving, and the
318  gyroscope data is dumped.
319
320  Camera movement is extracted from angle of deflection in gyroscope
321  movement. Test is a PASS if principal point in lens intrinsics
322  changes upon camera movement.
323  """
324
325  def test_lens_intrinsic_calibration(self):
326    rot_rig = {}
327    log_path = self.log_path
328
329    with its_session_utils.ItsSession(
330        device_id=self.dut.serial,
331        camera_id=self.camera_id,
332        hidden_physical_id=self.hidden_physical_id) as cam:
333
334      props = cam.get_camera_properties()
335      props = cam.override_with_hidden_physical_camera_props(props)
336
337      # Check if OIS supported
338      camera_properties_utils.skip_unless(
339          camera_properties_utils.optical_stabilization_supported(props))
340
341      # Initialize rotation rig
342      rot_rig['cntl'] = self.rotator_cntl
343      rot_rig['ch'] = self.rotator_ch
344      if rot_rig['cntl'].lower() != 'arduino':
345        raise AssertionError(
346            f'You must use the arduino controller for {_NAME}.')
347
348      largest_area = get_largest_video_size(cam, self.camera_id)
349
350      preview_size = preview_processing_utils.get_max_preview_test_size(
351          cam, self.camera_id, aspect_ratio=None, max_tested_area=largest_area)
352      logging.debug('preview_test_size: %s', preview_size)
353
354      recording_obj = preview_processing_utils.collect_data(
355          cam, self.tablet_device, preview_size, False,
356          rot_rig=rot_rig, ois=True)
357
358      # Get gyro events
359      logging.debug('Reading out inertial sensor events')
360      gyro_events = cam.get_sensor_events()['gyro']
361      logging.debug('Number of gyro samples %d', len(gyro_events))
362
363      # Grab the video from the save location on DUT
364      self.dut.adb.pull([recording_obj['recordedOutputPath'], log_path])
365
366      intrinsic_result = verify_lens_intrinsics(
367          recording_obj, gyro_events, _NAME, log_path)
368
369      # Don't change print to logging. Used for KPI.
370      print(f'{_NAME}_max_principal_point_diff: ',
371            intrinsic_result['max_pp_diff'])
372      # Assert PASS/FAIL criteria
373      if intrinsic_result['failure']:
374        first_api_level = its_session_utils.get_first_api_level(self.dut.serial)
375        failure_msg = intrinsic_result['failure']
376        if first_api_level >= its_session_utils.ANDROID15_API_LEVEL:
377          raise AssertionError(failure_msg)
378        else:
379          raise AssertionError(f'{its_session_utils.NOT_YET_MANDATED_MESSAGE}'
380                               f'\n\n{failure_msg}')
381
382      samples_results = verify_lens_intrinsics_sample(recording_obj)
383      if samples_results['failure']:
384        raise AssertionError(samples_results['failure'])
385
386      camera_properties_utils.skip_unless(
387          not (intrinsic_result['skip'] and samples_results['skip']),
388          'Lens intrinsic and samples are not available in results.')
389
390
391if __name__ == '__main__':
392  test_runner.main()
393
394