1# Copyright 2014 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import os 16import os.path 17import tempfile 18import subprocess 19import time 20import sys 21import textwrap 22import its.device 23 24def main(): 25 """Run all the automated tests, saving intermediate files, and producing 26 a summary/report of the results. 27 28 Script should be run from the top-level CameraITS directory. 29 """ 30 31 SKIP_RET_CODE = 101 32 33 # Not yet mandated tests 34 NOT_YET_MANDATED = { 35 "scene0":[ 36 "test_jitter" 37 ], 38 "scene1":[ 39 "test_ae_precapture_trigger", 40 "test_crop_region_raw", 41 "test_ev_compensation_advanced", 42 "test_ev_compensation_basic", 43 "test_yuv_plus_jpeg" 44 ], 45 "scene2":[], 46 "scene3":[] 47 } 48 49 # Get all the scene0 and scene1 tests, which can be run using the same 50 # physical setup. 51 scenes = ["scene0", "scene1", "scene2", "scene3"] 52 scene_req = { 53 "scene0" : None, 54 "scene1" : "A grey card covering at least the middle 30% of the scene", 55 "scene2" : "A picture containing human faces", 56 "scene3" : "A chart containing sharp edges like ISO 12233" 57 } 58 tests = [] 59 for d in scenes: 60 tests += [(d,s[:-3],os.path.join("tests", d, s)) 61 for s in os.listdir(os.path.join("tests",d)) 62 if s[-3:] == ".py"] 63 tests.sort() 64 65 # Make output directories to hold the generated files. 66 topdir = tempfile.mkdtemp() 67 print "Saving output files to:", topdir, "\n" 68 69 device_id = its.device.get_device_id() 70 device_id_arg = "device=" + device_id 71 print "Testing device " + device_id 72 73 camera_ids = [] 74 for s in sys.argv[1:]: 75 if s[:7] == "camera=" and len(s) > 7: 76 camera_ids.append(s[7:]) 77 78 # user doesn't specify camera id, run through all cameras 79 if not camera_ids: 80 camera_ids_path = os.path.join(topdir, "camera_ids.txt") 81 out_arg = "out=" + camera_ids_path 82 cmd = ['python', 83 os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg, 84 device_id_arg] 85 retcode = subprocess.call(cmd,cwd=topdir) 86 assert(retcode == 0) 87 with open(camera_ids_path, "r") as f: 88 for line in f: 89 camera_ids.append(line.replace('\n', '')) 90 91 print "Running ITS on the following cameras:", camera_ids 92 93 for camera_id in camera_ids: 94 # Loop capturing images until user confirm test scene is correct 95 camera_id_arg = "camera=" + camera_id 96 print "Preparing to run ITS on camera", camera_id 97 98 os.mkdir(os.path.join(topdir, camera_id)) 99 for d in scenes: 100 os.mkdir(os.path.join(topdir, camera_id, d)) 101 102 print "Start running ITS on camera: ", camera_id 103 # Run each test, capturing stdout and stderr. 104 summary = "ITS test result summary for camera " + camera_id + "\n" 105 numpass = 0 106 numskip = 0 107 numnotmandatedfail = 0 108 numfail = 0 109 110 prev_scene = "" 111 for (scene,testname,testpath) in tests: 112 if scene != prev_scene and scene_req[scene] != None: 113 out_path = os.path.join(topdir, camera_id, scene+".jpg") 114 out_arg = "out=" + out_path 115 scene_arg = "scene=" + scene_req[scene] 116 cmd = ['python', 117 os.path.join(os.getcwd(),"tools/validate_scene.py"), 118 camera_id_arg, out_arg, scene_arg, device_id_arg] 119 retcode = subprocess.call(cmd,cwd=topdir) 120 assert(retcode == 0) 121 print "Start running tests for", scene 122 prev_scene = scene 123 cmd = ['python', os.path.join(os.getcwd(),testpath)] + \ 124 sys.argv[1:] + [camera_id_arg] 125 outdir = os.path.join(topdir,camera_id,scene) 126 outpath = os.path.join(outdir,testname+"_stdout.txt") 127 errpath = os.path.join(outdir,testname+"_stderr.txt") 128 t0 = time.time() 129 with open(outpath,"w") as fout, open(errpath,"w") as ferr: 130 retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir) 131 t1 = time.time() 132 133 if retcode == 0: 134 retstr = "PASS " 135 numpass += 1 136 elif retcode == SKIP_RET_CODE: 137 retstr = "SKIP " 138 numskip += 1 139 elif retcode != 0 and testname in NOT_YET_MANDATED[scene]: 140 retstr = "FAIL*" 141 numnotmandatedfail += 1 142 else: 143 retstr = "FAIL " 144 numfail += 1 145 146 msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0) 147 print msg 148 summary += msg + "\n" 149 if retcode != 0 and retcode != SKIP_RET_CODE: 150 # Dump the stderr if the test fails 151 with open (errpath, "r") as error_file: 152 errors = error_file.read() 153 summary += errors + "\n" 154 155 if numskip > 0: 156 skipstr = ", %d test%s skipped" % (numskip, "s" if numskip > 1 else "") 157 else: 158 skipstr = "" 159 160 test_result = "\n%d / %d tests passed (%.1f%%)%s" % ( 161 numpass + numnotmandatedfail, len(tests) - numskip, 162 100.0 * float(numpass + numnotmandatedfail) / (len(tests) - numskip) 163 if len(tests) != numskip else 100.0, 164 skipstr) 165 print test_result 166 summary += test_result + "\n" 167 168 if numnotmandatedfail > 0: 169 msg = "(*) tests are not yet mandated" 170 print msg 171 summary += msg + "\n" 172 173 result = numfail == 0 174 print "Reporting ITS result to CtsVerifier" 175 summary_path = os.path.join(topdir, camera_id, "summary.txt") 176 with open(summary_path, "w") as f: 177 f.write(summary) 178 its.device.report_result(device_id, camera_id, result, summary_path) 179 180 print "ITS tests finished. Please go back to CtsVerifier and proceed" 181 182if __name__ == '__main__': 183 main() 184