1#!/usr/bin/env python3 2 3"""Sanity checks for test data. 4 5This program contains a class for traversing test cases that can be used 6independently of the checks. 7""" 8 9# Copyright The Mbed TLS Contributors 10# SPDX-License-Identifier: Apache-2.0 11# 12# Licensed under the Apache License, Version 2.0 (the "License"); you may 13# not use this file except in compliance with the License. 14# You may obtain a copy of the License at 15# 16# http://www.apache.org/licenses/LICENSE-2.0 17# 18# Unless required by applicable law or agreed to in writing, software 19# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 20# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21# See the License for the specific language governing permissions and 22# limitations under the License. 23 24import argparse 25import glob 26import os 27import re 28import sys 29 30class Results: 31 """Store file and line information about errors or warnings in test suites.""" 32 33 def __init__(self, options): 34 self.errors = 0 35 self.warnings = 0 36 self.ignore_warnings = options.quiet 37 38 def error(self, file_name, line_number, fmt, *args): 39 sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). 40 format(file_name, line_number, *args)) 41 self.errors += 1 42 43 def warning(self, file_name, line_number, fmt, *args): 44 if not self.ignore_warnings: 45 sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') 46 .format(file_name, line_number, *args)) 47 self.warnings += 1 48 49class TestDescriptionExplorer: 50 """An iterator over test cases with descriptions. 51 52The test cases that have descriptions are: 53* Individual unit tests (entries in a .data file) in test suites. 54* Individual test cases in ssl-opt.sh. 55 56This is an abstract class. To use it, derive a class that implements 57the process_test_case method, and call walk_all(). 58""" 59 60 def process_test_case(self, per_file_state, 61 file_name, line_number, description): 62 """Process a test case. 63 64per_file_state: an object created by new_per_file_state() at the beginning 65 of each file. 66file_name: a relative path to the file containing the test case. 67line_number: the line number in the given file. 68description: the test case description as a byte string. 69""" 70 raise NotImplementedError 71 72 def new_per_file_state(self): 73 """Return a new per-file state object. 74 75The default per-file state object is None. Child classes that require per-file 76state may override this method. 77""" 78 #pylint: disable=no-self-use 79 return None 80 81 def walk_test_suite(self, data_file_name): 82 """Iterate over the test cases in the given unit test data file.""" 83 in_paragraph = False 84 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 85 with open(data_file_name, 'rb') as data_file: 86 for line_number, line in enumerate(data_file, 1): 87 line = line.rstrip(b'\r\n') 88 if not line: 89 in_paragraph = False 90 continue 91 if line.startswith(b'#'): 92 continue 93 if not in_paragraph: 94 # This is a test case description line. 95 self.process_test_case(descriptions, 96 data_file_name, line_number, line) 97 in_paragraph = True 98 99 def walk_ssl_opt_sh(self, file_name): 100 """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" 101 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 102 with open(file_name, 'rb') as file_contents: 103 for line_number, line in enumerate(file_contents, 1): 104 # Assume that all run_test calls have the same simple form 105 # with the test description entirely on the same line as the 106 # function name. 107 m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) 108 if not m: 109 continue 110 description = m.group(1) 111 self.process_test_case(descriptions, 112 file_name, line_number, description) 113 114 @staticmethod 115 def collect_test_directories(): 116 """Get the relative path for the TLS and Crypto test directories.""" 117 if os.path.isdir('tests'): 118 tests_dir = 'tests' 119 elif os.path.isdir('suites'): 120 tests_dir = '.' 121 elif os.path.isdir('../suites'): 122 tests_dir = '..' 123 directories = [tests_dir] 124 return directories 125 126 def walk_all(self): 127 """Iterate over all named test cases.""" 128 test_directories = self.collect_test_directories() 129 for directory in test_directories: 130 for data_file_name in glob.glob(os.path.join(directory, 'suites', 131 '*.data')): 132 self.walk_test_suite(data_file_name) 133 ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') 134 if os.path.exists(ssl_opt_sh): 135 self.walk_ssl_opt_sh(ssl_opt_sh) 136 for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases', 137 '*.sh')): 138 self.walk_ssl_opt_sh(ssl_opt_file_name) 139 140class DescriptionChecker(TestDescriptionExplorer): 141 """Check all test case descriptions. 142 143* Check that each description is valid (length, allowed character set, etc.). 144* Check that there is no duplicated description inside of one test suite. 145""" 146 147 def __init__(self, results): 148 self.results = results 149 150 def new_per_file_state(self): 151 """Dictionary mapping descriptions to their line number.""" 152 return {} 153 154 def process_test_case(self, per_file_state, 155 file_name, line_number, description): 156 """Check test case descriptions for errors.""" 157 results = self.results 158 seen = per_file_state 159 if description in seen: 160 results.error(file_name, line_number, 161 'Duplicate description (also line {})', 162 seen[description]) 163 return 164 if re.search(br'[\t;]', description): 165 results.error(file_name, line_number, 166 'Forbidden character \'{}\' in description', 167 re.search(br'[\t;]', description).group(0).decode('ascii')) 168 if re.search(br'[^ -~]', description): 169 results.error(file_name, line_number, 170 'Non-ASCII character in description') 171 if len(description) > 66: 172 results.warning(file_name, line_number, 173 'Test description too long ({} > 66)', 174 len(description)) 175 seen[description] = line_number 176 177def main(): 178 parser = argparse.ArgumentParser(description=__doc__) 179 parser.add_argument('--quiet', '-q', 180 action='store_true', 181 help='Hide warnings') 182 parser.add_argument('--verbose', '-v', 183 action='store_false', dest='quiet', 184 help='Show warnings (default: on; undoes --quiet)') 185 options = parser.parse_args() 186 results = Results(options) 187 checker = DescriptionChecker(results) 188 checker.walk_all() 189 if (results.warnings or results.errors) and not options.quiet: 190 sys.stderr.write('{}: {} errors, {} warnings\n' 191 .format(sys.argv[0], results.errors, results.warnings)) 192 sys.exit(1 if results.errors else 0) 193 194if __name__ == '__main__': 195 main() 196