• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3"""
4This script generates autotest control files for dEQP. It supports
51) Generate control files for tests with Passing expectations.
62) Generate control files to run tests that are not passing.
73) Decomposing a test into shards. Ideally shard_count is chosen such that
8   each shard will run less than 1 minute. It mostly makes sense in
9   combination with "hasty".
10"""
11import os
12from collections import namedtuple
13# Use 'sudo pip install enum34' to install.
14from enum import Enum
15# Use 'sudo pip install jinja2' to install.
16from jinja2 import Template
17
18Test = namedtuple('Test', 'filter, suite, shards, time, hasty, tag, test_file, perf_failure_description')
19
20
21ATTRIBUTES_BVT_CQ = (
22    'suite:deqp, suite:graphics_per-day, suite:graphics_system, suite:bvt-inline')
23ATTRIBUTES_BVT_PB = (
24    'suite:deqp, suite:graphics_per-day, suite:graphics_system, '
25    'suite:bvt-perbuild'
26)
27ATTRIBUTES_DAILY = 'suite:deqp, suite:graphics_per-day, suite:graphics_system'
28
29class Suite(Enum):
30    none = 1
31    daily = 2
32    bvtcq = 3
33    bvtpb = 4
34
35test_file_folder = '/usr/local/deqp/master/'
36BVT_MASTER_FILE = '/usr/local/autotest/tests/graphics_dEQP/master/bvt.txt'
37GLES2_MASTER_FILE = os.path.join(test_file_folder, 'gles2-master.txt')
38GLES3_MASTER_FILE = os.path.join(test_file_folder, 'gles3-master.txt')
39GLES31_MASTER_FILE = os.path.join(test_file_folder, 'gles31-master.txt')
40VK_MASTER_FILE = os.path.join(test_file_folder, 'vk-master.txt')
41
42# List of tests' filter that should not append 'hasty' to its name.
43hasty_exclude_list = ['dEQP-VK-master']
44
45tests = [
46    Test('bvt',                    Suite.bvtcq, shards=1,  hasty=False, time='FAST',     tag='bvt',           test_file=BVT_MASTER_FILE,    perf_failure_description='Failures_BVT'),
47    Test('dEQP-GLES2-master',      Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles2-master',  test_file=GLES2_MASTER_FILE,  perf_failure_description='Failures_GLES2'),
48    Test('dEQP-GLES2-master',      Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles2-master',  test_file=GLES2_MASTER_FILE,  perf_failure_description=None),
49    # The stress, accuracy and performance tests are not part of -master lists.
50    # Hence we create control files in case we want to run them. But there is
51    # no strict requirement to keep them passing.
52    Test('dEQP-GLES2.stress',      Suite.daily, shards=1,  hasty=False, time='LONG',     tag='stress',        test_file=None,               perf_failure_description=None),
53    Test('dEQP-GLES3.accuracy',    Suite.none,  shards=1,  hasty=False, time='FAST',     tag=None,            test_file=None,               perf_failure_description=None),
54    Test('dEQP-GLES3-master',      Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles3-master',  test_file=GLES3_MASTER_FILE,  perf_failure_description='Failures_GLES3'),
55    Test('dEQP-GLES3-master',      Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles3-master',  test_file=GLES3_MASTER_FILE,  perf_failure_description=None),
56    Test('dEQP-GLES3.performance', Suite.none,  shards=1,  hasty=False, time='LONG',     tag=None,            test_file=None,               perf_failure_description=None),
57    # It is not worth running GLES3.stress in addition to GLES2.stress and GLES31.stress just to find stability issues.
58    Test('dEQP-GLES3.stress',      Suite.none,  shards=1,  hasty=False, time='LONG',     tag=None,            test_file=None,               perf_failure_description=None),
59    Test('dEQP-GLES31-master',     Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles31-master', test_file=GLES31_MASTER_FILE, perf_failure_description='Failures_GLES31'),
60    Test('dEQP-GLES31-master',     Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles31-master', test_file=GLES31_MASTER_FILE, perf_failure_description=None),
61    Test('dEQP-GLES31.stress',     Suite.daily, shards=1,  hasty=False, time='LONG',     tag='stress',        test_file=None,               perf_failure_description=None),
62    Test('dEQP-VK-master',         Suite.bvtpb, shards=1,  hasty=True,  time='LENGTHY',  tag='vk-master',     test_file=VK_MASTER_FILE,     perf_failure_description='Failures_VK'),
63]
64
65CONTROLFILE_TEMPLATE = Template(
66"""\
67# Copyright 2015 The Chromium OS Authors. All rights reserved.
68# Use of this source code is governed by a BSD-style license that can be
69# found in the LICENSE file.
70
71# Please do not edit this file! It has been created by generate_controlfiles.py.
72
73NAME = '{{testname}}'
74AUTHOR = 'chromeos-gfx'
75PURPOSE = 'Run the drawElements Quality Program test suite.'
76CRITERIA = 'All of the individual tests must pass.'
77ATTRIBUTES = '{{attributes}}'
78TIME = '{{time}}'
79TEST_CATEGORY = 'Functional'
80TEST_CLASS = 'graphics'
81TEST_TYPE = 'client'
82MAX_RESULT_SIZE_KB = 131072
83DOC = \"\"\"
84This test runs the drawElements Quality Program test suite.
85\"\"\"
86job.run_test('graphics_dEQP',{% if tag != None %}
87             tag = '{{tag}}',{% endif %}
88             opts = args + [
89                 {% if test_file == None %}'filter={{filter}}',
90                 'subset_to_run={{subset}}',
91                 {% else %}'test_names_file={{test_file}}',
92                 {% endif %}'hasty={{hasty}}',
93                 {% if perf_failure_description %}'perf_failure_description={{perf_failure_description}}',
94                 {% endif %}'shard_number={{shard}}',
95                 'shard_count={{shards}}'
96             ])"""
97    )
98
99#Unlike the normal version it batches many tests in a single run
100#to reduce testing time. Unfortunately this is less robust and
101#can lead to spurious failures.
102
103
104def get_controlfilename(test, shard=0):
105    return 'control.%s' % get_name(test, shard)
106
107
108def get_attributes(test):
109    if test.suite == Suite.bvtcq:
110        return ATTRIBUTES_BVT_CQ
111    if test.suite == Suite.bvtpb:
112        return ATTRIBUTES_BVT_PB
113    if test.suite == Suite.daily:
114        return ATTRIBUTES_DAILY
115    return ''
116
117
118def get_time(test):
119    return test.time
120
121
122def get_name(test, shard):
123    name = test.filter.replace('dEQP-', '', 1).lower()
124    if test.hasty and test.filter not in hasty_exclude_list:
125        name = '%s.hasty' % name
126    if test.shards > 1:
127        name = '%s.%d' % (name, shard)
128    return name
129
130
131def get_testname(test, shard=0):
132    return 'graphics_dEQP.%s' % get_name(test, shard)
133
134
135def write_controlfile(filename, content):
136    print 'Writing %s.' % filename
137    with open(filename, 'w+') as f:
138        f.write(content)
139
140
141def write_controlfiles(test):
142    attributes = get_attributes(test)
143    time = get_time(test)
144
145    for shard in xrange(0, test.shards):
146        testname = get_testname(test, shard)
147        filename = get_controlfilename(test, shard)
148        content = CONTROLFILE_TEMPLATE.render(
149            testname=testname,
150            attributes=attributes,
151            time=time,
152            filter=test.filter,
153            subset='Pass',
154            hasty=test.hasty,
155            shard=shard,
156            shards=test.shards,
157            test_file=test.test_file,
158            tag=test.tag,
159            perf_failure_description=test.perf_failure_description
160        )
161        write_controlfile(filename, content)
162
163
164def main():
165    for test in tests:
166        write_controlfiles(test)
167
168if __name__ == "__main__":
169    main()
170