• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import os
16import os.path
17import tempfile
18import subprocess
19import time
20import sys
21import textwrap
22import its.device
23
24def main():
25    """Run all the automated tests, saving intermediate files, and producing
26    a summary/report of the results.
27
28    Script should be run from the top-level CameraITS directory.
29    """
30
31    SKIP_RET_CODE = 101
32
33    # Not yet mandated tests
34    NOT_YET_MANDATED = {
35        "scene0":[
36            "test_jitter"
37        ],
38        "scene1":[
39            "test_ae_precapture_trigger",
40            "test_crop_region_raw",
41            "test_ev_compensation_advanced",
42            "test_ev_compensation_basic",
43            "test_yuv_plus_jpeg"
44        ],
45        "scene2":[],
46        "scene3":[]
47    }
48
49    # Get all the scene0 and scene1 tests, which can be run using the same
50    # physical setup.
51    scenes = ["scene0", "scene1", "scene2", "scene3"]
52    scene_req = {
53        "scene0" : None,
54        "scene1" : "A grey card covering at least the middle 30% of the scene",
55        "scene2" : "A picture containing human faces",
56        "scene3" : "A chart containing sharp edges like ISO 12233"
57    }
58    tests = []
59    for d in scenes:
60        tests += [(d,s[:-3],os.path.join("tests", d, s))
61                  for s in os.listdir(os.path.join("tests",d))
62                  if s[-3:] == ".py"]
63    tests.sort()
64
65    # Make output directories to hold the generated files.
66    topdir = tempfile.mkdtemp()
67    print "Saving output files to:", topdir, "\n"
68
69    device_id = its.device.get_device_id()
70    device_id_arg = "device=" + device_id
71    print "Testing device " + device_id
72
73    camera_ids = []
74    for s in sys.argv[1:]:
75        if s[:7] == "camera=" and len(s) > 7:
76            camera_ids.append(s[7:])
77
78    # user doesn't specify camera id, run through all cameras
79    if not camera_ids:
80        camera_ids_path = os.path.join(topdir, "camera_ids.txt")
81        out_arg = "out=" + camera_ids_path
82        cmd = ['python',
83               os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg,
84               device_id_arg]
85        retcode = subprocess.call(cmd,cwd=topdir)
86        assert(retcode == 0)
87        with open(camera_ids_path, "r") as f:
88            for line in f:
89                camera_ids.append(line.replace('\n', ''))
90
91    print "Running ITS on the following cameras:", camera_ids
92
93    for camera_id in camera_ids:
94        # Loop capturing images until user confirm test scene is correct
95        camera_id_arg = "camera=" + camera_id
96        print "Preparing to run ITS on camera", camera_id
97
98        os.mkdir(os.path.join(topdir, camera_id))
99        for d in scenes:
100            os.mkdir(os.path.join(topdir, camera_id, d))
101
102        print "Start running ITS on camera: ", camera_id
103        # Run each test, capturing stdout and stderr.
104        summary = "ITS test result summary for camera " + camera_id + "\n"
105        numpass = 0
106        numskip = 0
107        numnotmandatedfail = 0
108        numfail = 0
109
110        prev_scene = ""
111        for (scene,testname,testpath) in tests:
112            if scene != prev_scene and scene_req[scene] != None:
113                out_path = os.path.join(topdir, camera_id, scene+".jpg")
114                out_arg = "out=" + out_path
115                scene_arg = "scene=" + scene_req[scene]
116                cmd = ['python',
117                        os.path.join(os.getcwd(),"tools/validate_scene.py"),
118                        camera_id_arg, out_arg, scene_arg, device_id_arg]
119                retcode = subprocess.call(cmd,cwd=topdir)
120                assert(retcode == 0)
121                print "Start running tests for", scene
122            prev_scene = scene
123            cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
124                  sys.argv[1:] + [camera_id_arg]
125            outdir = os.path.join(topdir,camera_id,scene)
126            outpath = os.path.join(outdir,testname+"_stdout.txt")
127            errpath = os.path.join(outdir,testname+"_stderr.txt")
128            t0 = time.time()
129            with open(outpath,"w") as fout, open(errpath,"w") as ferr:
130                retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir)
131            t1 = time.time()
132
133            if retcode == 0:
134                retstr = "PASS "
135                numpass += 1
136            elif retcode == SKIP_RET_CODE:
137                retstr = "SKIP "
138                numskip += 1
139            elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
140                retstr = "FAIL*"
141                numnotmandatedfail += 1
142            else:
143                retstr = "FAIL "
144                numfail += 1
145
146            msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
147            print msg
148            summary += msg + "\n"
149            if retcode != 0 and retcode != SKIP_RET_CODE:
150                # Dump the stderr if the test fails
151                with open (errpath, "r") as error_file:
152                    errors = error_file.read()
153                    summary += errors + "\n"
154
155        if numskip > 0:
156            skipstr = ", %d test%s skipped" % (numskip, "s" if numskip > 1 else "")
157        else:
158            skipstr = ""
159
160        test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
161                numpass + numnotmandatedfail, len(tests) - numskip,
162                100.0 * float(numpass + numnotmandatedfail) / (len(tests) - numskip)
163                    if len(tests) != numskip else 100.0,
164                skipstr)
165        print test_result
166        summary += test_result + "\n"
167
168        if numnotmandatedfail > 0:
169            msg = "(*) tests are not yet mandated"
170            print msg
171            summary += msg + "\n"
172
173        result = numfail == 0
174        print "Reporting ITS result to CtsVerifier"
175        summary_path = os.path.join(topdir, camera_id, "summary.txt")
176        with open(summary_path, "w") as f:
177            f.write(summary)
178        its.device.report_result(device_id, camera_id, result, summary_path)
179
180    print "ITS tests finished. Please go back to CtsVerifier and proceed"
181
182if __name__ == '__main__':
183    main()
184