1#!/usr/bin/env python 2 3""" 4This script generates autotest control files for dEQP. It supports 51) Generate control files for tests with Passing expectations. 62) Generate control files to run tests that are not passing. 73) Decomposing a test into shards. Ideally shard_count is chosen such that 8 each shard will run less than 1 minute. It mostly makes sense in 9 combination with "hasty". 10""" 11import os 12from collections import namedtuple 13# Use 'sudo pip install enum34' to install. 14from enum import Enum 15# Use 'sudo pip install jinja2' to install. 16from jinja2 import Template 17 18Test = namedtuple('Test', 'filter, suite, shards, time, hasty, tag, test_file, perf_failure_description') 19 20 21ATTRIBUTES_BVT_CQ = ( 22 'suite:deqp, suite:graphics_per-day, suite:graphics_system, suite:bvt-inline') 23ATTRIBUTES_BVT_PB = ( 24 'suite:deqp, suite:graphics_per-day, suite:graphics_system, ' 25 'suite:bvt-perbuild' 26) 27ATTRIBUTES_DAILY = 'suite:deqp, suite:graphics_per-day, suite:graphics_system' 28 29class Suite(Enum): 30 none = 1 31 daily = 2 32 bvtcq = 3 33 bvtpb = 4 34 35test_file_folder = '/usr/local/deqp/master/' 36BVT_MASTER_FILE = '/usr/local/autotest/tests/graphics_dEQP/master/bvt.txt' 37GLES2_MASTER_FILE = os.path.join(test_file_folder, 'gles2-master.txt') 38GLES3_MASTER_FILE = os.path.join(test_file_folder, 'gles3-master.txt') 39GLES31_MASTER_FILE = os.path.join(test_file_folder, 'gles31-master.txt') 40VK_MASTER_FILE = os.path.join(test_file_folder, 'vk-master.txt') 41 42# List of tests' filter that should not append 'hasty' to its name. 43hasty_exclude_list = ['dEQP-VK-master'] 44 45tests = [ 46 Test('bvt', Suite.bvtcq, shards=1, hasty=False, time='FAST', tag='bvt', test_file=BVT_MASTER_FILE, perf_failure_description='Failures_BVT'), 47 Test('dEQP-GLES2-master', Suite.daily, shards=1, hasty=False, time='LENGTHY', tag='gles2-master', test_file=GLES2_MASTER_FILE, perf_failure_description='Failures_GLES2'), 48 # As we are following tot with dEQP the hasty shards have too much noise that is impossible to expect. 49 #Test('dEQP-GLES2-master', Suite.bvtpb, shards=10, hasty=True, time='FAST', tag='gles2-master', test_file=GLES2_MASTER_FILE, perf_failure_description=None), 50 # The stress, accuracy and performance tests are not part of -master lists. 51 # Hence we create control files in case we want to run them. But there is 52 # no strict requirement to keep them passing. 53 Test('dEQP-GLES2.stress', Suite.daily, shards=1, hasty=False, time='LONG', tag='stress', test_file=None, perf_failure_description=None), 54 Test('dEQP-GLES3.accuracy', Suite.none, shards=1, hasty=False, time='FAST', tag=None, test_file=None, perf_failure_description=None), 55 Test('dEQP-GLES3-master', Suite.daily, shards=1, hasty=False, time='LENGTHY', tag='gles3-master', test_file=GLES3_MASTER_FILE, perf_failure_description='Failures_GLES3'), 56 #Test('dEQP-GLES3-master', Suite.bvtpb, shards=10, hasty=True, time='FAST', tag='gles3-master', test_file=GLES3_MASTER_FILE, perf_failure_description=None), 57 Test('dEQP-GLES3.performance', Suite.none, shards=1, hasty=False, time='LONG', tag=None, test_file=None, perf_failure_description=None), 58 # It is not worth running GLES3.stress in addition to GLES2.stress and GLES31.stress just to find stability issues. 59 Test('dEQP-GLES3.stress', Suite.none, shards=1, hasty=False, time='LONG', tag=None, test_file=None, perf_failure_description=None), 60 Test('dEQP-GLES31-master', Suite.daily, shards=1, hasty=False, time='LENGTHY', tag='gles31-master', test_file=GLES31_MASTER_FILE, perf_failure_description='Failures_GLES31'), 61 #Test('dEQP-GLES31-master', Suite.bvtpb, shards=10, hasty=True, time='FAST', tag='gles31-master', test_file=GLES31_MASTER_FILE, perf_failure_description=None), 62 Test('dEQP-GLES31.stress', Suite.daily, shards=1, hasty=False, time='LONG', tag='stress', test_file=None, perf_failure_description=None), 63 Test('dEQP-VK-master', Suite.daily, shards=1, hasty=True, time='LENGTHY', tag='vk-master', test_file=VK_MASTER_FILE, perf_failure_description='Failures_VK'), 64] 65 66CONTROLFILE_TEMPLATE = Template( 67"""\ 68# Copyright 2015 The Chromium OS Authors. All rights reserved. 69# Use of this source code is governed by a BSD-style license that can be 70# found in the LICENSE file. 71 72# Please do not edit this file! It has been created by generate_controlfiles.py. 73 74NAME = '{{testname}}' 75AUTHOR = 'chromeos-gfx' 76PURPOSE = 'Run the drawElements Quality Program test suite.' 77CRITERIA = 'All of the individual tests must pass.' 78ATTRIBUTES = '{{attributes}}' 79TIME = '{{time}}' 80TEST_CATEGORY = 'Functional' 81TEST_CLASS = 'graphics' 82TEST_TYPE = 'client' 83MAX_RESULT_SIZE_KB = 131072 84DOC = \"\"\" 85This test runs the drawElements Quality Program test suite. 86\"\"\" 87job.run_test('graphics_dEQP',{% if tag != None %} 88 tag = '{{tag}}',{% endif %} 89 opts = args + [ 90 {% if test_file == None %}'filter={{filter}}', 91 'subset_to_run={{subset}}', 92 {% else %}'test_names_file={{test_file}}', 93 {% endif %}'hasty={{hasty}}', 94 {% if perf_failure_description %}'perf_failure_description={{perf_failure_description}}', 95 {% endif %}'shard_number={{shard}}', 96 'shard_count={{shards}}' 97 ])""" 98 ) 99 100#Unlike the normal version it batches many tests in a single run 101#to reduce testing time. Unfortunately this is less robust and 102#can lead to spurious failures. 103 104 105def get_controlfilename(test, shard=0): 106 return 'control.%s' % get_name(test, shard) 107 108 109def get_attributes(test): 110 if test.suite == Suite.bvtcq: 111 return ATTRIBUTES_BVT_CQ 112 if test.suite == Suite.bvtpb: 113 return ATTRIBUTES_BVT_PB 114 if test.suite == Suite.daily: 115 return ATTRIBUTES_DAILY 116 return '' 117 118 119def get_time(test): 120 return test.time 121 122 123def get_name(test, shard): 124 name = test.filter.replace('dEQP-', '', 1).lower() 125 if test.hasty and test.filter not in hasty_exclude_list: 126 name = '%s.hasty' % name 127 if test.shards > 1: 128 name = '%s.%d' % (name, shard) 129 return name 130 131 132def get_testname(test, shard=0): 133 return 'graphics_dEQP.%s' % get_name(test, shard) 134 135 136def write_controlfile(filename, content): 137 print 'Writing %s.' % filename 138 with open(filename, 'w+') as f: 139 f.write(content) 140 141 142def write_controlfiles(test): 143 attributes = get_attributes(test) 144 time = get_time(test) 145 146 for shard in xrange(0, test.shards): 147 testname = get_testname(test, shard) 148 filename = get_controlfilename(test, shard) 149 content = CONTROLFILE_TEMPLATE.render( 150 testname=testname, 151 attributes=attributes, 152 time=time, 153 filter=test.filter, 154 subset='Pass', 155 hasty=test.hasty, 156 shard=shard, 157 shards=test.shards, 158 test_file=test.test_file, 159 tag=test.tag, 160 perf_failure_description=test.perf_failure_description 161 ) 162 write_controlfile(filename, content) 163 164 165def main(): 166 for test in tests: 167 write_controlfiles(test) 168 169if __name__ == "__main__": 170 main() 171