1# Copyright 2014 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import math 17import os 18import os.path 19import re 20import subprocess 21import sys 22import tempfile 23import threading 24import time 25 26import its.caps 27import its.cv2image 28import its.device 29from its.device import ItsSession 30import its.image 31import rotation_rig as rot 32 33# For checking the installed APK's target SDK version 34MIN_SUPPORTED_SDK_VERSION = 28 # P 35 36CHART_DELAY = 1 # seconds 37CHART_LEVEL = 96 38NOT_YET_MANDATED_ALL = 100 39NUM_TRYS = 2 40PROC_TIMEOUT_CODE = -101 # terminated process return -process_id 41PROC_TIMEOUT_TIME = 900 # timeout in seconds for a process (15 minutes) 42SCENE3_FILE = os.path.join(os.environ['CAMERA_ITS_TOP'], 'pymodules', 'its', 43 'test_images', 'ISO12233.png') 44SKIP_RET_CODE = 101 # note this must be same as tests/scene*/test_* 45VGA_HEIGHT = 480 46VGA_WIDTH = 640 47 48# All possible scenes 49# Notes on scene names: 50# scene*_1/2/... are same scene split to load balance run times for scenes 51# scene*_a/b/... are similar scenes that share one or more tests 52ALL_SCENES = ['scene0', 'scene1_1', 'scene1_2', 'scene2_a', 'scene2_b', 53 'scene2_c', 'scene2_d', 'scene2_e', 'scene3', 'scene4', 54 'scene5', 'scene6', 'sensor_fusion', 'scene_change'] 55 56# Scenes that are logically grouped and can be called as group 57GROUPED_SCENES = { 58 'scene1': ['scene1_1', 'scene1_2'], 59 'scene2': ['scene2_a', 'scene2_b', 'scene2_c', 'scene2_d', 'scene2_e'] 60} 61 62# Scenes that can be automated through tablet display 63AUTO_SCENES = ['scene0', 'scene1_1', 'scene1_2', 'scene2_a', 'scene2_b', 64 'scene2_c', 'scene2_d', 'scene2_e', 'scene3', 'scene4', 65 'scene6', 'scene_change'] 66 67SCENE_REQ = { 68 'scene0': None, 69 'scene1_1': 'A grey card covering at least the middle 30% of the scene', 70 'scene1_2': 'A grey card covering at least the middle 30% of the scene', 71 'scene2_a': 'The picture in tests/scene2_a.pdf with 3 faces', 72 'scene2_b': 'The picture in tests/scene2_b.pdf with 3 faces', 73 'scene2_c': 'The picture in tests/scene2_c.pdf with 3 faces', 74 'scene2_d': 'The picture in tests/scene2_d.pdf with 3 faces', 75 'scene2_e': 'The picture in tests/scene2_e.pdf with 3 faces', 76 'scene3': 'The ISO 12233 chart', 77 'scene4': 'A specific test page of a circle covering at least the ' 78 'middle 50% of the scene. See CameraITS.pdf section 2.3.4 ' 79 'for more details', 80 'scene5': 'Capture images with a diffuser attached to the camera. See ' 81 'CameraITS.pdf section 2.3.4 for more details', 82 'scene6': 'A specific test page of a grid of 9x5 circles circle ' 83 'middle 50% of the scene.', 84 'sensor_fusion': 'Rotating checkboard pattern. See ' 85 'sensor_fusion/SensorFusion.pdf for detailed ' 86 'instructions.\nNote that this test will be skipped ' 87 'on devices not supporting REALTIME camera timestamp.', 88 'scene_change': 'The picture in tests/scene_change.pdf with faces' 89} 90 91SCENE_EXTRA_ARGS = { 92 'scene5': ['doAF=False'] 93} 94 95# Not yet mandated tests ['test', first_api_level mandatory] 96# ie. ['test_test_patterns', 30] is MANDATED for first_api_level >= 30 97NOT_YET_MANDATED = { 98 'scene0': [ 99 ['test_test_patterns', 30], 100 ['test_tonemap_curve', 30] 101 ], 102 'scene1_1': [ 103 ['test_ae_precapture_trigger', 28], 104 ['test_channel_saturation', 29] 105 ], 106 'scene1_2': [], 107 'scene2_a': [ 108 ['test_jpeg_quality', 30] 109 ], 110 'scene2_b': [ 111 ['test_auto_per_frame_control', NOT_YET_MANDATED_ALL] 112 ], 113 'scene2_c': [], 114 'scene2_d': [ 115 ['test_num_faces', 30] 116 ], 117 'scene2_e': [ 118 ['test_num_faces', 30], 119 ['test_continuous_picture', 30] 120 ], 121 'scene3': [], 122 'scene4': [], 123 'scene5': [], 124 'scene6': [ 125 ['test_zoom', 30] 126 ], 127 'sensor_fusion': [], 128 'scene_change': [ 129 ['test_scene_change', 31] 130 ] 131} 132 133# Must match mHiddenPhysicalCameraSceneIds in ItsTestActivity.java 134HIDDEN_PHYSICAL_CAMERA_TESTS = { 135 'scene0': [ 136 'test_burst_capture', 137 'test_metadata', 138 'test_read_write', 139 'test_sensor_events', 140 'test_unified_timestamps' 141 ], 142 'scene1_1': [ 143 'test_exposure', 144 'test_dng_noise_model', 145 'test_linearity', 146 ], 147 'scene1_2': [ 148 'test_raw_exposure', 149 'test_raw_sensitivity' 150 ], 151 'scene2_a': [ 152 'test_faces', 153 'test_num_faces' 154 ], 155 'scene2_b': [], 156 'scene2_c': [], 157 'scene2_d': [], 158 'scene2_e': [], 159 'scene3': [], 160 'scene4': [ 161 'test_aspect_ratio_and_crop' 162 ], 163 'scene5': [], 164 'scene6': [], 165 'sensor_fusion': [ 166 'test_sensor_fusion' 167 ], 168 'scene_change': [] 169} 170 171# Tests run in more than 1 scene. 172# List is created of type ['scene_source', 'test_to_be_repeated'] 173# for the test run in current scene. 174REPEATED_TESTS = { 175 'scene0': [], 176 'scene1_1': [], 177 'scene1_2': [], 178 'scene2_a': [], 179 'scene2_b': [ 180 ['scene2_a', 'test_num_faces'] 181 ], 182 'scene2_c': [ 183 ['scene2_a', 'test_num_faces'] 184 ], 185 'scene2_d': [ 186 ['scene2_a', 'test_num_faces'] 187 ], 188 'scene2_e': [ 189 ['scene2_a', 'test_num_faces'] 190 ], 191 'scene3': [], 192 'scene4': [], 193 'scene5': [], 194 'scene6': [], 195 'sensor_fusion': [], 196 'scene_change': [] 197} 198 199 200def determine_not_yet_mandated_tests(device_id): 201 """Determine from NEW_YET_MANDATED & phone info not_yet_mandated tests. 202 203 Args: 204 device_id: string of device id number 205 206 Returns: 207 dict of not yet mandated tests 208 """ 209 # initialize not_yet_mandated 210 not_yet_mandated = {} 211 for scene in ALL_SCENES: 212 not_yet_mandated[scene] = [] 213 214 # Determine first API level for device 215 first_api_level = its.device.get_first_api_level(device_id) 216 217 # Determine which scenes are not yet mandated for first api level 218 for scene, tests in NOT_YET_MANDATED.items(): 219 for test in tests: 220 if test[1] >= first_api_level: 221 not_yet_mandated[scene].append(test[0]) 222 return not_yet_mandated 223 224 225def expand_scene(scene, scenes): 226 """Expand a grouped scene and append its sub_scenes to scenes. 227 228 Args: 229 scene: scene in GROUPED_SCENES dict 230 scenes: list of scenes to append to 231 232 Returns: 233 updated scenes 234 """ 235 print 'Expanding %s to %s.' % (scene, str(GROUPED_SCENES[scene])) 236 for sub_scene in GROUPED_SCENES[scene]: 237 scenes.append(sub_scene) 238 239 240def run_subprocess_with_timeout(cmd, fout, ferr, outdir): 241 """Run subprocess with a timeout. 242 243 Args: 244 cmd: list containing python command 245 fout: stdout file for the test 246 ferr: stderr file for the test 247 outdir: dir location for fout/ferr 248 249 Returns: 250 process status or PROC_TIMEOUT_CODE if timer maxes 251 """ 252 253 proc = subprocess.Popen( 254 cmd, stdout=fout, stderr=ferr, cwd=outdir) 255 timer = threading.Timer(PROC_TIMEOUT_TIME, proc.kill) 256 257 try: 258 timer.start() 259 proc.communicate() 260 test_code = proc.returncode 261 finally: 262 timer.cancel() 263 264 if test_code < 0: 265 return PROC_TIMEOUT_CODE 266 else: 267 return test_code 268 269 270def calc_camera_fov(camera_id, hidden_physical_id): 271 """Determine the camera field of view from internal params.""" 272 with ItsSession(camera_id, hidden_physical_id) as cam: 273 props = cam.get_camera_properties() 274 props = cam.override_with_hidden_physical_camera_props(props) 275 focal_ls = props['android.lens.info.availableFocalLengths'] 276 if len(focal_ls) > 1: 277 print 'Doing capture to determine logical camera focal length' 278 cap = cam.do_capture(its.objects.auto_capture_request()) 279 focal_l = cap['metadata']['android.lens.focalLength'] 280 else: 281 focal_l = focal_ls[0] 282 sensor_size = props['android.sensor.info.physicalSize'] 283 diag = math.sqrt(sensor_size['height'] ** 2 + 284 sensor_size['width'] ** 2) 285 try: 286 fov = str(round(2 * math.degrees(math.atan(diag / (2 * focal_l))), 2)) 287 except ValueError: 288 fov = str(0) 289 print 'Calculated FoV: %s' % fov 290 return fov 291 292 293def evaluate_socket_failure(err_file_path): 294 """Determine if test fails due to socket FAIL.""" 295 socket_fail = False 296 with open(err_file_path, 'r') as ferr: 297 for line in ferr: 298 if (line.find('socket.error') != -1 or 299 line.find('socket.timeout') != -1 or 300 line.find('Problem with socket') != -1): 301 socket_fail = True 302 return socket_fail 303 304 305def run_rotations(camera_id, test_name): 306 """Determine if camera rotation is run for this test.""" 307 with ItsSession(camera_id) as cam: 308 props = cam.get_camera_properties() 309 method = {'test_sensor_fusion': { 310 'flag': its.caps.sensor_fusion_test_capable(props, cam), 311 'runs': 10}, 312 'test_multi_camera_frame_sync': { 313 'flag': its.caps.multi_camera_frame_sync_capable(props), 314 'runs': 5} 315 } 316 return method[test_name] 317 318 319def main(): 320 """Run all the automated tests, saving intermediate files, and producing 321 a summary/report of the results. 322 323 Script should be run from the top-level CameraITS directory. 324 325 Command line arguments: 326 camera: the camera(s) to be tested. Use comma to separate multiple 327 camera Ids. Ex: "camera=0,1" or "camera=1" 328 device: device id for adb 329 scenes: the test scene(s) to be executed. Use comma to separate 330 multiple scenes. Ex: "scenes=scene0,scene1_1" or 331 "scenes=0,1_1,sensor_fusion" (sceneX can be abbreviated by X 332 where X is scene name minus 'scene') 333 chart: another android device served as test chart display. 334 When this argument presents, change of test scene 335 will be handled automatically. Note that this argument 336 requires special physical/hardware setup to work and may not 337 work on all android devices. 338 result: Device ID to forward results to (in addition to the device 339 that the tests are running on). 340 rot_rig: ID of the rotation rig being used (formatted as 341 "<vendor ID>:<product ID>:<channel #>" or "default" for 342 Canakit-based rotators or "arduino:<channel #>" for 343 Arduino-based rotators) 344 tmp_dir: location of temp directory for output files 345 skip_scene_validation: force skip scene validation. Used when test scene 346 is setup up front and don't require tester validation. 347 dist: chart distance in cm. 348 """ 349 350 camera_id_combos = [] 351 scenes = [] 352 chart_host_id = None 353 result_device_id = None 354 rot_rig_id = None 355 tmp_dir = None 356 skip_scene_validation = False 357 chart_distance = its.cv2image.CHART_DISTANCE_RFOV 358 chart_level = CHART_LEVEL 359 one_camera_argv = sys.argv[1:] 360 361 for s in list(sys.argv[1:]): 362 if s[:7] == 'camera=' and len(s) > 7: 363 camera_ids = s[7:].split(',') 364 camera_id_combos = its.device.parse_camera_ids(camera_ids) 365 one_camera_argv.remove(s) 366 elif s[:7] == 'scenes=' and len(s) > 7: 367 scenes = s[7:].split(',') 368 elif s[:6] == 'chart=' and len(s) > 6: 369 chart_host_id = s[6:] 370 elif s[:7] == 'result=' and len(s) > 7: 371 result_device_id = s[7:] 372 elif s[:8] == 'rot_rig=' and len(s) > 8: 373 rot_rig_id = s[8:] # valid values: 'default', '$VID:$PID:$CH', 374 # or 'arduino:$CH'. The default '$VID:$PID:$CH' is '04d8:fc73:1' 375 elif s[:8] == 'tmp_dir=' and len(s) > 8: 376 tmp_dir = s[8:] 377 elif s == 'skip_scene_validation': 378 skip_scene_validation = True 379 elif s[:5] == 'dist=' and len(s) > 5: 380 chart_distance = float(re.sub('cm', '', s[5:])) 381 elif s[:11] == 'brightness=' and len(s) > 11: 382 chart_level = s[11:] 383 384 chart_dist_arg = 'dist= ' + str(chart_distance) 385 chart_level_arg = 'brightness=' + str(chart_level) 386 auto_scene_switch = chart_host_id is not None 387 merge_result_switch = result_device_id is not None 388 389 # Run through all scenes if user does not supply one 390 possible_scenes = AUTO_SCENES if auto_scene_switch else ALL_SCENES 391 if not scenes: 392 scenes = possible_scenes 393 else: 394 # Validate user input scene names 395 valid_scenes = True 396 temp_scenes = [] 397 for s in scenes: 398 if s in possible_scenes: 399 temp_scenes.append(s) 400 elif GROUPED_SCENES.has_key(s): 401 expand_scene(s, temp_scenes) 402 else: 403 try: 404 # Try replace "X" to "sceneX" 405 scene_str = "scene" + s 406 if scene_str in possible_scenes: 407 temp_scenes.append(scene_str) 408 elif GROUPED_SCENES.has_key(scene_str): 409 expand_scene(scene_str, temp_scenes) 410 else: 411 valid_scenes = False 412 break 413 except ValueError: 414 valid_scenes = False 415 break 416 417 if not valid_scenes: 418 print 'Unknown scene specified:', s 419 assert False 420 # assign temp_scenes back to scenes and remove duplicates 421 scenes = sorted(set(temp_scenes), key=temp_scenes.index) 422 423 # Make output directories to hold the generated files. 424 topdir = tempfile.mkdtemp(dir=tmp_dir) 425 subprocess.call(['chmod', 'g+rx', topdir]) 426 print "Saving output files to:", topdir, "\n" 427 428 device_id = its.device.get_device_id() 429 device_id_arg = "device=" + device_id 430 print "Testing device " + device_id 431 432 # Check CtsVerifier SDK level 433 # Here we only do warning as there is no guarantee on pm dump output formt not changed 434 # Also sometimes it's intentional to run mismatched versions 435 cmd = "adb -s %s shell pm dump com.android.cts.verifier" % (device_id) 436 dump_path = os.path.join(topdir, 'CtsVerifier.txt') 437 with open(dump_path, 'w') as fout: 438 fout.write('ITS minimum supported SDK version is %d\n--\n' % (MIN_SUPPORTED_SDK_VERSION)) 439 fout.flush() 440 ret_code = subprocess.call(cmd.split(), stdout=fout) 441 442 if ret_code != 0: 443 print "Warning: cannot get CtsVerifier SDK version. Is CtsVerifier installed?" 444 445 ctsv_version = None 446 ctsv_version_name = None 447 with open(dump_path, 'r') as f: 448 target_sdk_found = False 449 version_name_found = False 450 for line in f: 451 match = re.search('targetSdk=([0-9]+)', line) 452 if match: 453 ctsv_version = int(match.group(1)) 454 target_sdk_found = True 455 match = re.search('versionName=([\S]+)$', line) 456 if match: 457 ctsv_version_name = match.group(1) 458 version_name_found = True 459 if target_sdk_found and version_name_found: 460 break 461 462 if ctsv_version is None: 463 print "Warning: cannot get CtsVerifier SDK version. Is CtsVerifier installed?" 464 elif ctsv_version < MIN_SUPPORTED_SDK_VERSION: 465 print "Warning: CtsVerifier version (%d) < ITS version (%d), is this intentional?" % ( 466 ctsv_version, MIN_SUPPORTED_SDK_VERSION) 467 else: 468 print "CtsVerifier targetSdk is", ctsv_version 469 if ctsv_version_name: 470 print "CtsVerifier version name is", ctsv_version_name 471 472 # Hard check on ItsService/host script version that should catch incompatible APK/script 473 with ItsSession() as cam: 474 cam.check_its_version_compatible() 475 476 # Correctness check for devices 477 device_bfp = its.device.get_device_fingerprint(device_id) 478 assert device_bfp is not None 479 480 if auto_scene_switch: 481 chart_host_bfp = its.device.get_device_fingerprint(chart_host_id) 482 assert chart_host_bfp is not None 483 484 if merge_result_switch: 485 result_device_bfp = its.device.get_device_fingerprint(result_device_id) 486 assert_err_msg = ('Cannot merge result to a different build, from ' 487 '%s to %s' % (device_bfp, result_device_bfp)) 488 assert device_bfp == result_device_bfp, assert_err_msg 489 490 # user doesn't specify camera id, run through all cameras 491 if not camera_id_combos: 492 with its.device.ItsSession() as cam: 493 camera_ids = cam.get_camera_ids() 494 camera_id_combos = its.device.parse_camera_ids(camera_ids) 495 496 print "Running ITS on camera: %s, scene %s" % (camera_id_combos, scenes) 497 498 if auto_scene_switch: 499 # merge_result only supports run_parallel_tests 500 if merge_result_switch and camera_ids[0] == "1": 501 print "Skip chart screen" 502 time.sleep(1) 503 else: 504 print "Waking up chart screen: ", chart_host_id 505 screen_id_arg = ("screen=%s" % chart_host_id) 506 cmd = ["python", os.path.join(os.environ["CAMERA_ITS_TOP"], "tools", 507 "wake_up_screen.py"), screen_id_arg, 508 chart_level_arg] 509 wake_code = subprocess.call(cmd) 510 assert wake_code == 0 511 512 for id_combo in camera_id_combos: 513 # Initialize test results 514 results = {} 515 result_key = ItsSession.RESULT_KEY 516 for s in ALL_SCENES: 517 results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED} 518 519 camera_fov = calc_camera_fov(id_combo.id, id_combo.sub_id) 520 id_combo_string = id_combo.id 521 has_hidden_sub_camera = id_combo.sub_id is not None 522 if has_hidden_sub_camera: 523 id_combo_string += ItsSession.CAMERA_ID_TOKENIZER + id_combo.sub_id 524 scenes = [scene for scene in scenes if HIDDEN_PHYSICAL_CAMERA_TESTS[scene]] 525 # Loop capturing images until user confirm test scene is correct 526 camera_id_arg = "camera=" + id_combo.id 527 print "Preparing to run ITS on camera", id_combo_string, "for scenes ", scenes 528 529 os.mkdir(os.path.join(topdir, id_combo_string)) 530 for d in scenes: 531 os.mkdir(os.path.join(topdir, id_combo_string, d)) 532 533 tot_tests = [] 534 tot_pass = 0 535 not_yet_mandated = determine_not_yet_mandated_tests(device_id) 536 for scene in scenes: 537 # unit is millisecond for execution time record in CtsVerifier 538 scene_start_time = int(round(time.time() * 1000)) 539 skip_code = None 540 tests = [(s[:-3], os.path.join('tests', scene, s)) 541 for s in os.listdir(os.path.join('tests', scene)) 542 if s[-3:] == '.py' and s[:4] == 'test'] 543 if REPEATED_TESTS[scene]: 544 for t in REPEATED_TESTS[scene]: 545 tests.append((t[1], os.path.join('tests', t[0], t[1]+'.py'))) 546 tests.sort() 547 tot_tests.extend(tests) 548 549 summary = 'Cam' + id_combo_string + ' ' + scene + '\n' 550 numpass = 0 551 numskip = 0 552 num_not_mandated_fail = 0 553 numfail = 0 554 validate_switch = True 555 if SCENE_REQ[scene] is not None: 556 out_path = os.path.join(topdir, id_combo_string, scene+'.jpg') 557 out_arg = 'out=' + out_path 558 if ((scene == 'sensor_fusion' and rot_rig_id) or 559 skip_scene_validation): 560 validate_switch = False 561 cmd = None 562 if auto_scene_switch: 563 if (not merge_result_switch or 564 (merge_result_switch and id_combo_string == '0')): 565 scene_arg = 'scene=' + scene 566 fov_arg = 'fov=' + camera_fov 567 cmd = ['python', 568 os.path.join(os.getcwd(), 'tools/load_scene.py'), 569 scene_arg, chart_dist_arg, fov_arg, screen_id_arg] 570 else: 571 time.sleep(CHART_DELAY) 572 else: 573 # Skip scene validation under certain conditions 574 if validate_switch and not merge_result_switch: 575 scene_arg = 'scene=' + SCENE_REQ[scene] 576 extra_args = SCENE_EXTRA_ARGS.get(scene, []) 577 cmd = ['python', 578 os.path.join(os.getcwd(), 579 'tools/validate_scene.py'), 580 camera_id_arg, out_arg, 581 scene_arg, device_id_arg] + extra_args 582 if cmd is not None: 583 valid_scene_code = subprocess.call(cmd, cwd=topdir) 584 assert valid_scene_code == 0 585 print 'Start running ITS on camera %s, %s' % ( 586 id_combo_string, scene) 587 # Extract chart from scene for scene3 once up front 588 chart_loc_arg = '' 589 chart_height = its.cv2image.CHART_HEIGHT 590 if scene == 'scene3': 591 chart_height *= its.cv2image.calc_chart_scaling( 592 chart_distance, camera_fov) 593 chart = its.cv2image.Chart(SCENE3_FILE, chart_height, 594 chart_distance, 595 its.cv2image.CHART_SCALE_START, 596 its.cv2image.CHART_SCALE_STOP, 597 its.cv2image.CHART_SCALE_STEP, 598 id_combo.id) 599 chart_loc_arg = 'chart_loc=%.2f,%.2f,%.2f,%.2f,%.3f' % ( 600 chart.xnorm, chart.ynorm, chart.wnorm, chart.hnorm, 601 chart.scale) 602 if scene == 'scene_change' and not auto_scene_switch: 603 print '\nWave hand over camera to create scene change' 604 # Run each test, capturing stdout and stderr. 605 for (testname, testpath) in tests: 606 # Only pick predefined tests for hidden physical camera 607 if has_hidden_sub_camera and \ 608 testname not in HIDDEN_PHYSICAL_CAMERA_TESTS[scene]: 609 numskip += 1 610 continue 611 if auto_scene_switch: 612 if merge_result_switch and id_combo_string == '0': 613 # Send an input event to keep the screen not dimmed. 614 # Since we are not using camera of chart screen, FOCUS event 615 # should do nothing but keep the screen from dimming. 616 # The "sleep after x minutes of inactivity" display setting 617 # determines how long this command can keep screen bright. 618 # Setting it to something like 30 minutes should be enough. 619 cmd = ('adb -s %s shell input keyevent FOCUS' 620 % chart_host_id) 621 subprocess.call(cmd.split()) 622 t0 = time.time() 623 t_rotate = 0.0 # time in seconds 624 for num_try in range(NUM_TRYS): 625 outdir = os.path.join(topdir, id_combo_string, scene) 626 outpath = os.path.join(outdir, testname+'_stdout.txt') 627 errpath = os.path.join(outdir, testname+'_stderr.txt') 628 if scene == 'sensor_fusion': 629 # determine if you need to rotate for specific test 630 rotation_props = run_rotations(id_combo.id, testname) 631 if rotation_props['flag']: 632 if rot_rig_id: 633 print 'Rotating phone w/ rig %s' % rot_rig_id 634 rig = 'python tools/rotation_rig.py rotator=%s num_rotations=%s' % ( 635 rot_rig_id, rotation_props['runs']) 636 subprocess.Popen(rig.split()) 637 t_rotate = (rotation_props['runs'] * 638 len(rot.ARDUINO_ANGLES) * 639 rot.ARDUINO_MOVE_TIME) + 2 # 2s slop 640 else: 641 print 'Rotate phone 15s as shown in SensorFusion.pdf' 642 else: 643 test_code = skip_code 644 if skip_code is not SKIP_RET_CODE: 645 cmd = ['python', os.path.join(os.getcwd(), testpath)] 646 cmd += one_camera_argv + ["camera="+id_combo_string] + [chart_loc_arg] 647 cmd += [chart_dist_arg] 648 with open(outpath, 'w') as fout, open(errpath, 'w') as ferr: 649 test_code = run_subprocess_with_timeout( 650 cmd, fout, ferr, outdir) 651 if test_code == 0 or test_code == SKIP_RET_CODE: 652 break 653 else: 654 socket_fail = evaluate_socket_failure(errpath) 655 if socket_fail or test_code == PROC_TIMEOUT_CODE: 656 if num_try != NUM_TRYS-1: 657 print ' Retry %s/%s' % (scene, testname) 658 else: 659 break 660 else: 661 break 662 t_test = time.time() - t0 663 664 # define rotator_type 665 rotator_type = 'canakit' 666 if rot_rig_id and 'arduino' in rot_rig_id.split(':'): 667 rotator_type = 'arduino' 668 # if arduino, wait for rotations to stop 669 if t_rotate > t_test: 670 time.sleep(t_rotate - t_test) 671 672 test_failed = False 673 if test_code == 0: 674 retstr = "PASS " 675 numpass += 1 676 elif test_code == SKIP_RET_CODE: 677 retstr = "SKIP " 678 numskip += 1 679 elif test_code != 0 and testname in not_yet_mandated[scene]: 680 retstr = "FAIL*" 681 num_not_mandated_fail += 1 682 else: 683 retstr = "FAIL " 684 numfail += 1 685 test_failed = True 686 687 msg = '%s %s/%s' % (retstr, scene, testname) 688 if rotator_type == 'arduino': 689 msg += ' [%.1fs]' % t_rotate 690 else: 691 msg += ' [%.1fs]' % t_test 692 print msg 693 its.device.adb_log(device_id, msg) 694 msg_short = '%s %s [%.1fs]' % (retstr, testname, t_test) 695 if test_failed: 696 summary += msg_short + "\n" 697 698 # unit is millisecond for execution time record in CtsVerifier 699 scene_end_time = int(round(time.time() * 1000)) 700 701 if numskip > 0: 702 skipstr = ", %d test%s skipped" % ( 703 numskip, "s" if numskip > 1 else "") 704 else: 705 skipstr = "" 706 707 test_result = "\n%d / %d tests passed (%.1f%%)%s" % ( 708 numpass + num_not_mandated_fail, len(tests) - numskip, 709 100.0 * float(numpass + num_not_mandated_fail) / 710 (len(tests) - numskip) 711 if len(tests) != numskip else 100.0, skipstr) 712 print test_result 713 714 if num_not_mandated_fail > 0: 715 msg = "(*) tests are not yet mandated" 716 print msg 717 718 tot_pass += numpass 719 print "%s compatibility score: %.f/100\n" % ( 720 scene, 100.0 * numpass / len(tests)) 721 722 summary_path = os.path.join(topdir, id_combo_string, scene, "summary.txt") 723 with open(summary_path, "w") as f: 724 f.write(summary) 725 726 passed = numfail == 0 727 results[scene][result_key] = (ItsSession.RESULT_PASS if passed 728 else ItsSession.RESULT_FAIL) 729 results[scene][ItsSession.SUMMARY_KEY] = summary_path 730 results[scene][ItsSession.START_TIME_KEY] = scene_start_time 731 results[scene][ItsSession.END_TIME_KEY] = scene_end_time 732 733 if tot_tests: 734 print "Compatibility Score: %.f/100" % (100.0 * tot_pass / len(tot_tests)) 735 else: 736 print "Compatibility Score: 0/100" 737 738 msg = "Reporting ITS result to CtsVerifier" 739 print msg 740 its.device.adb_log(device_id, msg) 741 if merge_result_switch: 742 # results are modified by report_result 743 results_backup = copy.deepcopy(results) 744 its.device.report_result(result_device_id, id_combo_string, results_backup) 745 746 # Report hidden_physical_id results as well. 747 its.device.report_result(device_id, id_combo_string, results) 748 749 if auto_scene_switch: 750 if merge_result_switch: 751 print 'Skip shutting down chart screen' 752 else: 753 print 'Shutting down chart screen: ', chart_host_id 754 screen_id_arg = ('screen=%s' % chart_host_id) 755 cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 756 'toggle_screen.py'), screen_id_arg, 757 'state=OFF'] 758 screen_off_code = subprocess.call(cmd) 759 assert screen_off_code == 0 760 761 print 'Shutting down DUT screen: ', device_id 762 screen_id_arg = ('screen=%s' % device_id) 763 cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 764 'toggle_screen.py'), screen_id_arg, 765 'state=OFF'] 766 screen_off_code = subprocess.call(cmd) 767 assert screen_off_code == 0 768 769 print "ITS tests finished. Please go back to CtsVerifier and proceed" 770 771if __name__ == '__main__': 772 main() 773