• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3"""
4This script generates autotest control files for dEQP. It supports
51) Generate control files for tests with Passing expectations.
62) Generate control files to run tests that are not passing.
73) Decomposing a test into shards. Ideally shard_count is chosen such that
8   each shard will run less than 1 minute. It mostly makes sense in
9   combination with "hasty".
10"""
11import os
12from collections import namedtuple
13# Use 'sudo pip install enum34' to install.
14from enum import Enum
15# Use 'sudo pip install jinja2' to install.
16from jinja2 import Template
17
18Test = namedtuple('Test', 'filter, suite, shards, time, hasty, tag, test_file, perf_failure_description')
19
20
21ATTRIBUTES_BVT_CQ = (
22    'suite:deqp, suite:graphics_per-day, suite:graphics_system, suite:bvt-inline')
23ATTRIBUTES_BVT_PB = (
24    'suite:deqp, suite:graphics_per-day, suite:graphics_system, '
25    'suite:bvt-perbuild'
26)
27ATTRIBUTES_DAILY = 'suite:deqp, suite:graphics_per-day, suite:graphics_system'
28
29class Suite(Enum):
30    none = 1
31    daily = 2
32    bvtcq = 3
33    bvtpb = 4
34
35test_file_folder = '/usr/local/deqp/master/'
36BVT_MASTER_FILE = '/usr/local/autotest/tests/graphics_dEQP/master/bvt.txt'
37GLES2_MASTER_FILE = os.path.join(test_file_folder, 'gles2-master.txt')
38GLES3_MASTER_FILE = os.path.join(test_file_folder, 'gles3-master.txt')
39GLES31_MASTER_FILE = os.path.join(test_file_folder, 'gles31-master.txt')
40VK_MASTER_FILE = os.path.join(test_file_folder, 'vk-master.txt')
41
42# List of tests' filter that should not append 'hasty' to its name.
43hasty_exclude_list = ['dEQP-VK-master']
44
45tests = [
46    Test('bvt',                    Suite.bvtcq, shards=1,  hasty=False, time='FAST',     tag='bvt',           test_file=BVT_MASTER_FILE,    perf_failure_description='Failures_BVT'),
47    Test('dEQP-GLES2-master',      Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles2-master',  test_file=GLES2_MASTER_FILE,  perf_failure_description='Failures_GLES2'),
48    # As we are following tot with dEQP the hasty shards have too much noise that is impossible to expect.
49    #Test('dEQP-GLES2-master',      Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles2-master',  test_file=GLES2_MASTER_FILE,  perf_failure_description=None),
50    # The stress, accuracy and performance tests are not part of -master lists.
51    # Hence we create control files in case we want to run them. But there is
52    # no strict requirement to keep them passing.
53    Test('dEQP-GLES2.stress',      Suite.daily, shards=1,  hasty=False, time='LONG',     tag='stress',        test_file=None,               perf_failure_description=None),
54    Test('dEQP-GLES3.accuracy',    Suite.none,  shards=1,  hasty=False, time='FAST',     tag=None,            test_file=None,               perf_failure_description=None),
55    Test('dEQP-GLES3-master',      Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles3-master',  test_file=GLES3_MASTER_FILE,  perf_failure_description='Failures_GLES3'),
56    #Test('dEQP-GLES3-master',      Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles3-master',  test_file=GLES3_MASTER_FILE,  perf_failure_description=None),
57    Test('dEQP-GLES3.performance', Suite.none,  shards=1,  hasty=False, time='LONG',     tag=None,            test_file=None,               perf_failure_description=None),
58    # It is not worth running GLES3.stress in addition to GLES2.stress and GLES31.stress just to find stability issues.
59    Test('dEQP-GLES3.stress',      Suite.none,  shards=1,  hasty=False, time='LONG',     tag=None,            test_file=None,               perf_failure_description=None),
60    Test('dEQP-GLES31-master',     Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles31-master', test_file=GLES31_MASTER_FILE, perf_failure_description='Failures_GLES31'),
61    #Test('dEQP-GLES31-master',     Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles31-master', test_file=GLES31_MASTER_FILE, perf_failure_description=None),
62    Test('dEQP-GLES31.stress',     Suite.daily, shards=1,  hasty=False, time='LONG',     tag='stress',        test_file=None,               perf_failure_description=None),
63    Test('dEQP-VK-master',         Suite.daily, shards=1,  hasty=True,  time='LENGTHY',  tag='vk-master',     test_file=VK_MASTER_FILE,     perf_failure_description='Failures_VK'),
64]
65
66CONTROLFILE_TEMPLATE = Template(
67"""\
68# Copyright 2015 The Chromium OS Authors. All rights reserved.
69# Use of this source code is governed by a BSD-style license that can be
70# found in the LICENSE file.
71
72# Please do not edit this file! It has been created by generate_controlfiles.py.
73
74NAME = '{{testname}}'
75AUTHOR = 'chromeos-gfx'
76PURPOSE = 'Run the drawElements Quality Program test suite.'
77CRITERIA = 'All of the individual tests must pass.'
78ATTRIBUTES = '{{attributes}}'
79TIME = '{{time}}'
80TEST_CATEGORY = 'Functional'
81TEST_CLASS = 'graphics'
82TEST_TYPE = 'client'
83MAX_RESULT_SIZE_KB = 131072
84DOC = \"\"\"
85This test runs the drawElements Quality Program test suite.
86\"\"\"
87job.run_test('graphics_dEQP',{% if tag != None %}
88             tag = '{{tag}}',{% endif %}
89             opts = args + [
90                 {% if test_file == None %}'filter={{filter}}',
91                 'subset_to_run={{subset}}',
92                 {% else %}'test_names_file={{test_file}}',
93                 {% endif %}'hasty={{hasty}}',
94                 {% if perf_failure_description %}'perf_failure_description={{perf_failure_description}}',
95                 {% endif %}'shard_number={{shard}}',
96                 'shard_count={{shards}}'
97             ])"""
98    )
99
100#Unlike the normal version it batches many tests in a single run
101#to reduce testing time. Unfortunately this is less robust and
102#can lead to spurious failures.
103
104
105def get_controlfilename(test, shard=0):
106    return 'control.%s' % get_name(test, shard)
107
108
109def get_attributes(test):
110    if test.suite == Suite.bvtcq:
111        return ATTRIBUTES_BVT_CQ
112    if test.suite == Suite.bvtpb:
113        return ATTRIBUTES_BVT_PB
114    if test.suite == Suite.daily:
115        return ATTRIBUTES_DAILY
116    return ''
117
118
119def get_time(test):
120    return test.time
121
122
123def get_name(test, shard):
124    name = test.filter.replace('dEQP-', '', 1).lower()
125    if test.hasty and test.filter not in hasty_exclude_list:
126        name = '%s.hasty' % name
127    if test.shards > 1:
128        name = '%s.%d' % (name, shard)
129    return name
130
131
132def get_testname(test, shard=0):
133    return 'graphics_dEQP.%s' % get_name(test, shard)
134
135
136def write_controlfile(filename, content):
137    print 'Writing %s.' % filename
138    with open(filename, 'w+') as f:
139        f.write(content)
140
141
142def write_controlfiles(test):
143    attributes = get_attributes(test)
144    time = get_time(test)
145
146    for shard in xrange(0, test.shards):
147        testname = get_testname(test, shard)
148        filename = get_controlfilename(test, shard)
149        content = CONTROLFILE_TEMPLATE.render(
150            testname=testname,
151            attributes=attributes,
152            time=time,
153            filter=test.filter,
154            subset='Pass',
155            hasty=test.hasty,
156            shard=shard,
157            shards=test.shards,
158            test_file=test.test_file,
159            tag=test.tag,
160            perf_failure_description=test.perf_failure_description
161        )
162        write_controlfile(filename, content)
163
164
165def main():
166    for test in tests:
167        write_controlfiles(test)
168
169if __name__ == "__main__":
170    main()
171