• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2
3"""
4This script generates autotest control files for dEQP. It supports
51) Generate control files for tests with Passing expectations.
62) Generate control files to run tests that are not passing.
73) Decomposing a test into shards. Ideally shard_count is chosen such that
8   each shard will run less than 1 minute. It mostly makes sense in
9   combination with "hasty".
10"""
11import os
12from collections import namedtuple
13# Use 'sudo pip install enum34' to install.
14from enum import Enum
15# Use 'sudo pip install jinja2' to install.
16from jinja2 import Template
17
18Test = namedtuple('Test', 'filter, suite, shards, time, hasty, tag, test_file, perf_failure_description')
19
20
21ATTRIBUTES_BVT_PB = (
22    'suite:deqp, suite:graphics_per-day, suite:graphics_system, '
23    'suite:bvt-perbuild'
24)
25ATTRIBUTES_DAILY = 'suite:deqp, suite:graphics_per-day, suite:graphics_system'
26
27class Suite(Enum):
28    none = 1
29    daily = 2
30    bvtcq = 3
31    bvtpb = 4
32
33test_file_folder = '/usr/local/deqp/master/'
34BVT_MASTER_FILE = '/usr/local/autotest/tests/graphics_dEQP/master/bvt.txt'
35GLES2_MASTER_FILE = os.path.join(test_file_folder, 'gles2-master.txt')
36GLES3_MASTER_FILE = os.path.join(test_file_folder, 'gles3-master.txt')
37GLES31_MASTER_FILE = os.path.join(test_file_folder, 'gles31-master.txt')
38VK_MASTER_FILE = os.path.join(test_file_folder, 'vk-master.txt')
39
40# List of tests' filter that should not append 'hasty' to its name.
41hasty_exclude_list = ['dEQP-VK-master']
42
43tests = [
44    Test('bvt',                    Suite.daily, shards=1,  hasty=False, time='FAST',     tag='bvt',           test_file=BVT_MASTER_FILE,    perf_failure_description='Failures_BVT'),
45    Test('dEQP-GLES2-master',      Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles2-master',  test_file=GLES2_MASTER_FILE,  perf_failure_description='Failures_GLES2'),
46    # As we are following tot with dEQP the hasty shards have too much noise that is impossible to expect.
47    #Test('dEQP-GLES2-master',      Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles2-master',  test_file=GLES2_MASTER_FILE,  perf_failure_description=None),
48    # The stress, accuracy and performance tests are not part of -master lists.
49    # Hence we create control files in case we want to run them. But there is
50    # no strict requirement to keep them passing.
51    Test('dEQP-GLES3.accuracy',    Suite.none,  shards=1,  hasty=False, time='FAST',     tag=None,            test_file=None,               perf_failure_description=None),
52    Test('dEQP-GLES3-master',      Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles3-master',  test_file=GLES3_MASTER_FILE,  perf_failure_description='Failures_GLES3'),
53    #Test('dEQP-GLES3-master',      Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles3-master',  test_file=GLES3_MASTER_FILE,  perf_failure_description=None),
54    Test('dEQP-GLES3.performance', Suite.none,  shards=1,  hasty=False, time='LONG',     tag=None,            test_file=None,               perf_failure_description=None),
55    # It is not worth running GLES3.stress in addition to GLES2.stress and GLES31.stress just to find stability issues.
56    Test('dEQP-GLES31-master',     Suite.daily, shards=1,  hasty=False, time='LENGTHY',  tag='gles31-master', test_file=GLES31_MASTER_FILE, perf_failure_description='Failures_GLES31'),
57    #Test('dEQP-GLES31-master',     Suite.bvtpb, shards=10, hasty=True,  time='FAST',     tag='gles31-master', test_file=GLES31_MASTER_FILE, perf_failure_description=None),
58    Test('dEQP-VK-master',         Suite.daily, shards=1,  hasty=True,  time='LENGTHY',  tag='vk-master',     test_file=VK_MASTER_FILE,     perf_failure_description='Failures_VK'),
59]
60
61CONTROLFILE_TEMPLATE = Template(
62"""\
63# Copyright 2015 The Chromium OS Authors. All rights reserved.
64# Use of this source code is governed by a BSD-style license that can be
65# found in the LICENSE file.
66
67# Please do not edit this file! It has been created by generate_controlfiles.py.
68
69NAME = '{{testname}}'
70AUTHOR = 'chromeos-gfx'
71PURPOSE = 'Run the drawElements Quality Program test suite.'
72CRITERIA = 'All of the individual tests must pass.'
73ATTRIBUTES = '{{attributes}}'
74TIME = '{{time}}'
75TEST_CATEGORY = 'Functional'
76TEST_CLASS = 'graphics'
77TEST_TYPE = 'client'
78MAX_RESULT_SIZE_KB = 131072
79DOC = \"\"\"
80This test runs the drawElements Quality Program test suite.
81\"\"\"
82job.run_test('graphics_dEQP',{% if tag != None %}
83             tag = '{{tag}}',{% endif %}
84             opts = args + [
85                 {% if test_file == None %}'filter={{filter}}',
86                 'subset_to_run={{subset}}',
87                 {% else %}'test_names_file={{test_file}}',
88                 {% endif %}'hasty={{hasty}}',
89                 {% if perf_failure_description %}'perf_failure_description={{perf_failure_description}}',
90                 {% endif %}'shard_number={{shard}}',
91                 'shard_count={{shards}}'
92             ])"""
93    )
94
95#Unlike the normal version it batches many tests in a single run
96#to reduce testing time. Unfortunately this is less robust and
97#can lead to spurious failures.
98
99
100def get_controlfilename(test, shard=0):
101    return 'control.%s' % get_name(test, shard)
102
103
104def get_attributes(test):
105    if test.suite == Suite.bvtpb:
106        return ATTRIBUTES_BVT_PB
107    if test.suite == Suite.daily:
108        return ATTRIBUTES_DAILY
109    return ''
110
111
112def get_time(test):
113    return test.time
114
115
116def get_name(test, shard):
117    name = test.filter.replace('dEQP-', '', 1).lower()
118    if test.hasty and test.filter not in hasty_exclude_list:
119        name = '%s.hasty' % name
120    if test.shards > 1:
121        name = '%s.%d' % (name, shard)
122    return name
123
124
125def get_testname(test, shard=0):
126    return 'graphics_dEQP.%s' % get_name(test, shard)
127
128
129def write_controlfile(filename, content):
130    print 'Writing %s.' % filename
131    with open(filename, 'w+') as f:
132        f.write(content)
133
134
135def write_controlfiles(test):
136    attributes = get_attributes(test)
137    time = get_time(test)
138
139    for shard in xrange(0, test.shards):
140        testname = get_testname(test, shard)
141        filename = get_controlfilename(test, shard)
142        content = CONTROLFILE_TEMPLATE.render(
143            testname=testname,
144            attributes=attributes,
145            time=time,
146            filter=test.filter,
147            subset='Pass',
148            hasty=test.hasty,
149            shard=shard,
150            shards=test.shards,
151            test_file=test.test_file,
152            tag=test.tag,
153            perf_failure_description=test.perf_failure_description
154        )
155        write_controlfile(filename, content)
156
157
158def main():
159    for test in tests:
160        write_controlfiles(test)
161
162if __name__ == "__main__":
163    main()
164