1#!/usr/bin/env python3 2 3"""Sanity checks for test data. 4 5This program contains a class for traversing test cases that can be used 6independently of the checks. 7""" 8 9# Copyright The Mbed TLS Contributors 10# SPDX-License-Identifier: Apache-2.0 11# 12# Licensed under the Apache License, Version 2.0 (the "License"); you may 13# not use this file except in compliance with the License. 14# You may obtain a copy of the License at 15# 16# http://www.apache.org/licenses/LICENSE-2.0 17# 18# Unless required by applicable law or agreed to in writing, software 19# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 20# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21# See the License for the specific language governing permissions and 22# limitations under the License. 23 24import argparse 25import glob 26import os 27import re 28import sys 29 30class Results: 31 """Store file and line information about errors or warnings in test suites.""" 32 33 def __init__(self, options): 34 self.errors = 0 35 self.warnings = 0 36 self.ignore_warnings = options.quiet 37 38 def error(self, file_name, line_number, fmt, *args): 39 sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). 40 format(file_name, line_number, *args)) 41 self.errors += 1 42 43 def warning(self, file_name, line_number, fmt, *args): 44 if not self.ignore_warnings: 45 sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') 46 .format(file_name, line_number, *args)) 47 self.warnings += 1 48 49class TestDescriptionExplorer: 50 """An iterator over test cases with descriptions. 51 52The test cases that have descriptions are: 53* Individual unit tests (entries in a .data file) in test suites. 54* Individual test cases in ssl-opt.sh. 55 56This is an abstract class. To use it, derive a class that implements 57the process_test_case method, and call walk_all(). 58""" 59 60 def process_test_case(self, per_file_state, 61 file_name, line_number, description): 62 """Process a test case. 63 64per_file_state: an object created by new_per_file_state() at the beginning 65 of each file. 66file_name: a relative path to the file containing the test case. 67line_number: the line number in the given file. 68description: the test case description as a byte string. 69""" 70 raise NotImplementedError 71 72 def new_per_file_state(self): 73 """Return a new per-file state object. 74 75The default per-file state object is None. Child classes that require per-file 76state may override this method. 77""" 78 #pylint: disable=no-self-use 79 return None 80 81 def walk_test_suite(self, data_file_name): 82 """Iterate over the test cases in the given unit test data file.""" 83 in_paragraph = False 84 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 85 with open(data_file_name, 'rb') as data_file: 86 for line_number, line in enumerate(data_file, 1): 87 line = line.rstrip(b'\r\n') 88 if not line: 89 in_paragraph = False 90 continue 91 if line.startswith(b'#'): 92 continue 93 if not in_paragraph: 94 # This is a test case description line. 95 self.process_test_case(descriptions, 96 data_file_name, line_number, line) 97 in_paragraph = True 98 99 def walk_ssl_opt_sh(self, file_name): 100 """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" 101 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 102 with open(file_name, 'rb') as file_contents: 103 for line_number, line in enumerate(file_contents, 1): 104 # Assume that all run_test calls have the same simple form 105 # with the test description entirely on the same line as the 106 # function name. 107 m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) 108 if not m: 109 continue 110 description = m.group(1) 111 self.process_test_case(descriptions, 112 file_name, line_number, description) 113 114 @staticmethod 115 def collect_test_directories(): 116 """Get the relative path for the TLS and Crypto test directories.""" 117 if os.path.isdir('tests'): 118 tests_dir = 'tests' 119 elif os.path.isdir('suites'): 120 tests_dir = '.' 121 elif os.path.isdir('../suites'): 122 tests_dir = '..' 123 directories = [tests_dir] 124 return directories 125 126 def walk_all(self): 127 """Iterate over all named test cases.""" 128 test_directories = self.collect_test_directories() 129 for directory in test_directories: 130 for data_file_name in glob.glob(os.path.join(directory, 'suites', 131 '*.data')): 132 self.walk_test_suite(data_file_name) 133 ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') 134 if os.path.exists(ssl_opt_sh): 135 self.walk_ssl_opt_sh(ssl_opt_sh) 136 for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases', 137 '*.sh')): 138 self.walk_ssl_opt_sh(ssl_opt_file_name) 139 140class TestDescriptions(TestDescriptionExplorer): 141 """Collect the available test cases.""" 142 143 def __init__(self): 144 super().__init__() 145 self.descriptions = set() 146 147 def process_test_case(self, _per_file_state, 148 file_name, _line_number, description): 149 """Record an available test case.""" 150 base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) 151 key = ';'.join([base_name, description.decode('utf-8')]) 152 self.descriptions.add(key) 153 154def collect_available_test_cases(): 155 """Collect the available test cases.""" 156 explorer = TestDescriptions() 157 explorer.walk_all() 158 return sorted(explorer.descriptions) 159 160class DescriptionChecker(TestDescriptionExplorer): 161 """Check all test case descriptions. 162 163* Check that each description is valid (length, allowed character set, etc.). 164* Check that there is no duplicated description inside of one test suite. 165""" 166 167 def __init__(self, results): 168 self.results = results 169 170 def new_per_file_state(self): 171 """Dictionary mapping descriptions to their line number.""" 172 return {} 173 174 def process_test_case(self, per_file_state, 175 file_name, line_number, description): 176 """Check test case descriptions for errors.""" 177 results = self.results 178 seen = per_file_state 179 if description in seen: 180 results.error(file_name, line_number, 181 'Duplicate description (also line {})', 182 seen[description]) 183 return 184 if re.search(br'[\t;]', description): 185 results.error(file_name, line_number, 186 'Forbidden character \'{}\' in description', 187 re.search(br'[\t;]', description).group(0).decode('ascii')) 188 if re.search(br'[^ -~]', description): 189 results.error(file_name, line_number, 190 'Non-ASCII character in description') 191 if len(description) > 66: 192 results.warning(file_name, line_number, 193 'Test description too long ({} > 66)', 194 len(description)) 195 seen[description] = line_number 196 197def main(): 198 parser = argparse.ArgumentParser(description=__doc__) 199 parser.add_argument('--list-all', 200 action='store_true', 201 help='List all test cases, without doing checks') 202 parser.add_argument('--quiet', '-q', 203 action='store_true', 204 help='Hide warnings') 205 parser.add_argument('--verbose', '-v', 206 action='store_false', dest='quiet', 207 help='Show warnings (default: on; undoes --quiet)') 208 options = parser.parse_args() 209 if options.list_all: 210 descriptions = collect_available_test_cases() 211 sys.stdout.write('\n'.join(descriptions + [''])) 212 return 213 results = Results(options) 214 checker = DescriptionChecker(results) 215 checker.walk_all() 216 if (results.warnings or results.errors) and not options.quiet: 217 sys.stderr.write('{}: {} errors, {} warnings\n' 218 .format(sys.argv[0], results.errors, results.warnings)) 219 sys.exit(1 if results.errors else 0) 220 221if __name__ == '__main__': 222 main() 223