1#!/usr/bin/env python 2 3""" 4This script generates autotest control files for dEQP. It supports 51) Generate control files for tests with Passing expectations. 62) Generate control files to run tests that are not passing. 73) Decomposing a test into shards. Ideally shard_count is chosen such that 8 each shard will run less than 1 minute. It mostly makes sense in 9 combination with "hasty". 10""" 11 12from collections import namedtuple 13# Use 'sudo pip install enum34' to install. 14from enum import Enum 15 16Test = namedtuple('Test', 'filter, suite, shards, time, hasty, notpass') 17 18ATTRIBUTES_BVT_CQ = ( 19 'suite:deqp, suite:graphics_per-day, suite:graphics_system, suite:bvt-cq') 20ATTRIBUTES_BVT_PB = ( 21 'suite:deqp, suite:graphics_per-day, suite:graphics_system, ' 22 'suite:bvt-perbuild' 23) 24ATTRIBUTES_DAILY = 'suite:deqp, suite:graphics_per-day, suite:graphics_system' 25 26class Suite(Enum): 27 none = 1 28 daily = 2 29 bvtcq = 3 30 bvtpb = 4 31 32tests = [ 33 Test('dEQP-EGL.functional', Suite.none, shards=1, hasty=False, notpass=False, time='LENGTHY'), 34 Test('dEQP-EGL.info', Suite.none, shards=1, hasty=False, notpass=False, time='SHORT'), 35 Test('dEQP-EGL.performance', Suite.none, shards=1, hasty=False, notpass=False, time='SHORT'), 36 Test('dEQP-EGL.stress', Suite.none, shards=1, hasty=False, notpass=False, time='LONG'), 37 Test('dEQP-GLES2.accuracy', Suite.bvtcq, shards=1, hasty=False, notpass=False, time='FAST'), 38 Test('dEQP-GLES2.capability', Suite.bvtcq, shards=1, hasty=False, notpass=False, time='FAST'), 39 Test('dEQP-GLES2.functional', Suite.daily, shards=1, hasty=False, notpass=False, time='LENGTHY'), 40 Test('dEQP-GLES2.functional', Suite.none, shards=1, hasty=True, notpass=False, time='LONG'), 41 Test('dEQP-GLES2.functional', Suite.bvtpb, shards=10, hasty=True, notpass=False, time='FAST'), 42 Test('dEQP-GLES2.info', Suite.bvtcq, shards=1, hasty=False, notpass=False, time='FAST'), 43 Test('dEQP-GLES2.performance', Suite.daily, shards=1, hasty=False, notpass=False, time='LONG'), 44 Test('dEQP-GLES2.stress', Suite.daily, shards=1, hasty=False, notpass=False, time='LONG'), 45 Test('dEQP-GLES3.accuracy', Suite.bvtcq, shards=1, hasty=False, notpass=False, time='FAST'), 46 Test('dEQP-GLES3.functional', Suite.daily, shards=1, hasty=False, notpass=False, time='LENGTHY'), 47 Test('dEQP-GLES3.functional', Suite.none, shards=1, hasty=True, notpass=False, time='LONG'), 48 Test('dEQP-GLES3.functional', Suite.daily, shards=10, hasty=True, notpass=False, time='FAST'), 49 Test('dEQP-GLES3.info', Suite.bvtcq, shards=1, hasty=False, notpass=False, time='FAST'), 50 Test('dEQP-GLES3.performance', Suite.daily, shards=1, hasty=False, notpass=False, time='LONG'), 51 Test('dEQP-GLES3.stress', Suite.daily, shards=1, hasty=False, notpass=False, time='LONG'), 52 Test('dEQP-GLES31.functional', Suite.none, shards=1, hasty=False, notpass=False, time='LENGTHY'), 53 Test('dEQP-GLES31.functional', Suite.bvtpb, shards=10, hasty=True, notpass=False, time='FAST'), 54 Test('dEQP-GLES31.info', Suite.bvtcq, shards=1, hasty=False, notpass=False, time='FAST'), 55 Test('dEQP-GLES31.stress', Suite.none, shards=1, hasty=False, notpass=False, time='LONG'), 56 Test('dEQP-VK.api', Suite.none, shards=1, hasty=True, notpass=False, time='LONG'), 57 Test('dEQP-VK.api.smoke', Suite.bvtcq, shards=1, hasty=False, notpass=False, time='FAST'), 58 Test('dEQP-VK.binding_model', Suite.none, shards=1, hasty=True, notpass=False, time='LONG'), 59 Test('dEQP-VK.glsl', Suite.none, shards=1, hasty=True, notpass=False, time='LONG'), 60 Test('dEQP-VK.info', Suite.bvtcq, shards=1, hasty=False, notpass=False, time='FAST'), 61 Test('dEQP-VK.pipeline', Suite.none, shards=1, hasty=True, notpass=False, time='LONG'), 62 Test('dEQP-VK.spirv_assembly', Suite.none, shards=1, hasty=True, notpass=False, time='SHORT'), 63] 64 65CONTROLFILE_TEMPLATE = ( 66"""\ 67# Copyright 2015 The Chromium OS Authors. All rights reserved. 68# Use of this source code is governed by a BSD-style license that can be 69# found in the LICENSE file. 70 71# Please do not edit this file! It has been created by generate_controlfiles.py. 72 73NAME = '{0}' 74AUTHOR = 'chromeos-gfx' 75PURPOSE = 'Run the drawElements Quality Program test suite.' 76CRITERIA = 'All of the individual tests must pass.' 77ATTRIBUTES = '{1}' 78TIME = '{2}' 79{3}TEST_CATEGORY = 'Functional' 80TEST_CLASS = 'graphics' 81TEST_TYPE = 'client' 82DOC = \"\"\" 83This test runs the drawElements Quality Program test suite. 84\"\"\" 85 86job.run_test('graphics_dEQP', opts = args + ['filter={4}', 87 'subset_to_run={5}', 88 'hasty={6}', 89 'shard_number={7}', 90 'shard_count={8}'])""") 91 92#Unlike the normal version it batches many tests in a single run 93#to reduce testing time. Unfortunately this is less robust and 94#can lead to spurious failures. 95 96 97def get_controlfilename(test, shard=0): 98 return 'control.%s' % get_name(test, shard) 99 100def get_dependencies(test): 101 if test.notpass: 102 return "DEPENDENCIES = 'cleanup-reboot'\n" 103 return '' 104 105def get_attributes(test): 106 if test.suite == Suite.bvtcq: 107 return ATTRIBUTES_BVT_CQ 108 if test.suite == Suite.bvtpb: 109 return ATTRIBUTES_BVT_PB 110 if test.suite == Suite.daily: 111 return ATTRIBUTES_DAILY 112 return '' 113 114def get_time(test): 115 return test.time 116 117def get_name(test, shard): 118 name = test.filter.replace('dEQP-', '', 1).lower() 119 if test.hasty: 120 name = '%s.hasty' % name 121 if test.shards > 1: 122 name = '%s.%d' % (name, shard) 123 if test.notpass: 124 name = name + '.NotPass' 125 return name 126 127def get_testname(test, shard=0): 128 return 'graphics_dEQP.%s' % get_name(test, shard) 129 130def write_controlfile(filename, content): 131 print 'Writing %s.' % filename 132 with open(filename, 'w+') as f: 133 f.write(content) 134 135def write_controlfiles(test): 136 attributes = get_attributes(test) 137 time = get_time(test) 138 dependencies = get_dependencies(test) 139 if test.shards > 1: 140 for shard in xrange(0, test.shards): 141 subset = 'Pass' 142 testname = get_testname(test, shard) 143 filename = get_controlfilename(test, shard) 144 content = CONTROLFILE_TEMPLATE.format( 145 testname, attributes, time, dependencies, test.filter, subset, 146 test.hasty, shard, test.shards) 147 write_controlfile(filename, content) 148 else: 149 if test.notpass: 150 subset = 'NotPass' 151 testname = get_testname(test) 152 filename = get_controlfilename(test) 153 content = CONTROLFILE_TEMPLATE.format( 154 testname, attributes, time, dependencies, test.filter, subset, 155 test.hasty, 0, test.shards) 156 write_controlfile(filename, content) 157 test = Test(test.filter, 158 test.suite, 159 test.shards, 160 test.time, 161 test.hasty, 162 notpass=False) 163 dependencies = get_dependencies(test) 164 subset = 'Pass' 165 testname = get_testname(test) 166 filename = get_controlfilename(test) 167 content = CONTROLFILE_TEMPLATE.format(testname, attributes, time, 168 dependencies, test.filter, subset, 169 test.hasty, 0, test.shards) 170 write_controlfile(filename, content) 171 172 173for test in tests: 174 write_controlfiles(test) 175