1#!/usr/bin/env python3 2 3"""Sanity checks for test data. 4 5This program contains a class for traversing test cases that can be used 6independently of the checks. 7""" 8 9# Copyright The Mbed TLS Contributors 10# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 11 12import argparse 13import glob 14import os 15import re 16import subprocess 17import sys 18 19class Results: 20 """Store file and line information about errors or warnings in test suites.""" 21 22 def __init__(self, options): 23 self.errors = 0 24 self.warnings = 0 25 self.ignore_warnings = options.quiet 26 27 def error(self, file_name, line_number, fmt, *args): 28 sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). 29 format(file_name, line_number, *args)) 30 self.errors += 1 31 32 def warning(self, file_name, line_number, fmt, *args): 33 if not self.ignore_warnings: 34 sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') 35 .format(file_name, line_number, *args)) 36 self.warnings += 1 37 38class TestDescriptionExplorer: 39 """An iterator over test cases with descriptions. 40 41The test cases that have descriptions are: 42* Individual unit tests (entries in a .data file) in test suites. 43* Individual test cases in ssl-opt.sh. 44 45This is an abstract class. To use it, derive a class that implements 46the process_test_case method, and call walk_all(). 47""" 48 49 def process_test_case(self, per_file_state, 50 file_name, line_number, description): 51 """Process a test case. 52 53per_file_state: an object created by new_per_file_state() at the beginning 54 of each file. 55file_name: a relative path to the file containing the test case. 56line_number: the line number in the given file. 57description: the test case description as a byte string. 58""" 59 raise NotImplementedError 60 61 def new_per_file_state(self): 62 """Return a new per-file state object. 63 64The default per-file state object is None. Child classes that require per-file 65state may override this method. 66""" 67 #pylint: disable=no-self-use 68 return None 69 70 def walk_test_suite(self, data_file_name): 71 """Iterate over the test cases in the given unit test data file.""" 72 in_paragraph = False 73 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 74 with open(data_file_name, 'rb') as data_file: 75 for line_number, line in enumerate(data_file, 1): 76 line = line.rstrip(b'\r\n') 77 if not line: 78 in_paragraph = False 79 continue 80 if line.startswith(b'#'): 81 continue 82 if not in_paragraph: 83 # This is a test case description line. 84 self.process_test_case(descriptions, 85 data_file_name, line_number, line) 86 in_paragraph = True 87 88 def walk_ssl_opt_sh(self, file_name): 89 """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" 90 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 91 with open(file_name, 'rb') as file_contents: 92 for line_number, line in enumerate(file_contents, 1): 93 # Assume that all run_test calls have the same simple form 94 # with the test description entirely on the same line as the 95 # function name. 96 m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) 97 if not m: 98 continue 99 description = m.group(1) 100 self.process_test_case(descriptions, 101 file_name, line_number, description) 102 103 def walk_compat_sh(self, file_name): 104 """Iterate over the test cases compat.sh with a similar format.""" 105 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 106 compat_cmd = ['sh', file_name, '--list-test-case'] 107 compat_output = subprocess.check_output(compat_cmd) 108 # Assume compat.sh is responsible for printing identical format of 109 # test case description between --list-test-case and its OUTCOME.CSV 110 description = compat_output.strip().split(b'\n') 111 # idx indicates the number of test case since there is no line number 112 # in `compat.sh` for each test case. 113 for idx, descrip in enumerate(description): 114 self.process_test_case(descriptions, file_name, idx, descrip) 115 116 @staticmethod 117 def collect_test_directories(): 118 """Get the relative path for the TLS and Crypto test directories.""" 119 if os.path.isdir('tests'): 120 tests_dir = 'tests' 121 elif os.path.isdir('suites'): 122 tests_dir = '.' 123 elif os.path.isdir('../suites'): 124 tests_dir = '..' 125 directories = [tests_dir] 126 return directories 127 128 def walk_all(self): 129 """Iterate over all named test cases.""" 130 test_directories = self.collect_test_directories() 131 for directory in test_directories: 132 for data_file_name in glob.glob(os.path.join(directory, 'suites', 133 '*.data')): 134 self.walk_test_suite(data_file_name) 135 ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') 136 if os.path.exists(ssl_opt_sh): 137 self.walk_ssl_opt_sh(ssl_opt_sh) 138 compat_sh = os.path.join(directory, 'compat.sh') 139 if os.path.exists(compat_sh): 140 self.walk_compat_sh(compat_sh) 141 142class TestDescriptions(TestDescriptionExplorer): 143 """Collect the available test cases.""" 144 145 def __init__(self): 146 super().__init__() 147 self.descriptions = set() 148 149 def process_test_case(self, _per_file_state, 150 file_name, _line_number, description): 151 """Record an available test case.""" 152 base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) 153 key = ';'.join([base_name, description.decode('utf-8')]) 154 self.descriptions.add(key) 155 156def collect_available_test_cases(): 157 """Collect the available test cases.""" 158 explorer = TestDescriptions() 159 explorer.walk_all() 160 return sorted(explorer.descriptions) 161 162class DescriptionChecker(TestDescriptionExplorer): 163 """Check all test case descriptions. 164 165* Check that each description is valid (length, allowed character set, etc.). 166* Check that there is no duplicated description inside of one test suite. 167""" 168 169 def __init__(self, results): 170 self.results = results 171 172 def new_per_file_state(self): 173 """Dictionary mapping descriptions to their line number.""" 174 return {} 175 176 def process_test_case(self, per_file_state, 177 file_name, line_number, description): 178 """Check test case descriptions for errors.""" 179 results = self.results 180 seen = per_file_state 181 if description in seen: 182 results.error(file_name, line_number, 183 'Duplicate description (also line {})', 184 seen[description]) 185 return 186 if re.search(br'[\t;]', description): 187 results.error(file_name, line_number, 188 'Forbidden character \'{}\' in description', 189 re.search(br'[\t;]', description).group(0).decode('ascii')) 190 if re.search(br'[^ -~]', description): 191 results.error(file_name, line_number, 192 'Non-ASCII character in description') 193 if len(description) > 66: 194 results.warning(file_name, line_number, 195 'Test description too long ({} > 66)', 196 len(description)) 197 seen[description] = line_number 198 199def main(): 200 parser = argparse.ArgumentParser(description=__doc__) 201 parser.add_argument('--list-all', 202 action='store_true', 203 help='List all test cases, without doing checks') 204 parser.add_argument('--quiet', '-q', 205 action='store_true', 206 help='Hide warnings') 207 parser.add_argument('--verbose', '-v', 208 action='store_false', dest='quiet', 209 help='Show warnings (default: on; undoes --quiet)') 210 options = parser.parse_args() 211 if options.list_all: 212 descriptions = collect_available_test_cases() 213 sys.stdout.write('\n'.join(descriptions + [''])) 214 return 215 results = Results(options) 216 checker = DescriptionChecker(results) 217 checker.walk_all() 218 if (results.warnings or results.errors) and not options.quiet: 219 sys.stderr.write('{}: {} errors, {} warnings\n' 220 .format(sys.argv[0], results.errors, results.warnings)) 221 sys.exit(1 if results.errors else 0) 222 223if __name__ == '__main__': 224 main() 225