1#!/usr/bin/env python3 2"""Describe the test coverage of PSA functions in terms of return statuses. 3 41. Build Mbed Crypto with -DRECORD_PSA_STATUS_COVERAGE_LOG 52. Run psa_collect_statuses.py 6 7The output is a series of line of the form "psa_foo PSA_ERROR_XXX". Each 8function/status combination appears only once. 9 10This script must be run from the top of an Mbed Crypto source tree. 11The build command is "make -DRECORD_PSA_STATUS_COVERAGE_LOG", which is 12only supported with make (as opposed to CMake or other build methods). 13""" 14 15# Copyright The Mbed TLS Contributors 16# SPDX-License-Identifier: Apache-2.0 17# 18# Licensed under the Apache License, Version 2.0 (the "License"); you may 19# not use this file except in compliance with the License. 20# You may obtain a copy of the License at 21# 22# http://www.apache.org/licenses/LICENSE-2.0 23# 24# Unless required by applicable law or agreed to in writing, software 25# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 26# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27# See the License for the specific language governing permissions and 28# limitations under the License. 29 30import argparse 31import os 32import subprocess 33import sys 34 35DEFAULT_STATUS_LOG_FILE = 'tests/statuses.log' 36DEFAULT_PSA_CONSTANT_NAMES = 'programs/psa/psa_constant_names' 37 38class Statuses: 39 """Information about observed return statues of API functions.""" 40 41 def __init__(self): 42 self.functions = {} 43 self.codes = set() 44 self.status_names = {} 45 46 def collect_log(self, log_file_name): 47 """Read logs from RECORD_PSA_STATUS_COVERAGE_LOG. 48 49 Read logs produced by running Mbed Crypto test suites built with 50 -DRECORD_PSA_STATUS_COVERAGE_LOG. 51 """ 52 with open(log_file_name) as log: 53 for line in log: 54 value, function, tail = line.split(':', 2) 55 if function not in self.functions: 56 self.functions[function] = {} 57 fdata = self.functions[function] 58 if value not in self.functions[function]: 59 fdata[value] = [] 60 fdata[value].append(tail) 61 self.codes.add(int(value)) 62 63 def get_constant_names(self, psa_constant_names): 64 """Run psa_constant_names to obtain names for observed numerical values.""" 65 values = [str(value) for value in self.codes] 66 cmd = [psa_constant_names, 'status'] + values 67 output = subprocess.check_output(cmd).decode('ascii') 68 for value, name in zip(values, output.rstrip().split('\n')): 69 self.status_names[value] = name 70 71 def report(self): 72 """Report observed return values for each function. 73 74 The report is a series of line of the form "psa_foo PSA_ERROR_XXX". 75 """ 76 for function in sorted(self.functions.keys()): 77 fdata = self.functions[function] 78 names = [self.status_names[value] for value in fdata.keys()] 79 for name in sorted(names): 80 sys.stdout.write('{} {}\n'.format(function, name)) 81 82def collect_status_logs(options): 83 """Build and run unit tests and report observed function return statuses. 84 85 Build Mbed Crypto with -DRECORD_PSA_STATUS_COVERAGE_LOG, run the 86 test suites and display information about observed return statuses. 87 """ 88 rebuilt = False 89 if not options.use_existing_log and os.path.exists(options.log_file): 90 os.remove(options.log_file) 91 if not os.path.exists(options.log_file): 92 if options.clean_before: 93 subprocess.check_call(['make', 'clean'], 94 cwd='tests', 95 stdout=sys.stderr) 96 with open(os.devnull, 'w') as devnull: 97 make_q_ret = subprocess.call(['make', '-q', 'lib', 'tests'], 98 stdout=devnull, stderr=devnull) 99 if make_q_ret != 0: 100 subprocess.check_call(['make', 'RECORD_PSA_STATUS_COVERAGE_LOG=1'], 101 stdout=sys.stderr) 102 rebuilt = True 103 subprocess.check_call(['make', 'test'], 104 stdout=sys.stderr) 105 data = Statuses() 106 data.collect_log(options.log_file) 107 data.get_constant_names(options.psa_constant_names) 108 if rebuilt and options.clean_after: 109 subprocess.check_call(['make', 'clean'], 110 cwd='tests', 111 stdout=sys.stderr) 112 return data 113 114def main(): 115 parser = argparse.ArgumentParser(description=globals()['__doc__']) 116 parser.add_argument('--clean-after', 117 action='store_true', 118 help='Run "make clean" after rebuilding') 119 parser.add_argument('--clean-before', 120 action='store_true', 121 help='Run "make clean" before regenerating the log file)') 122 parser.add_argument('--log-file', metavar='FILE', 123 default=DEFAULT_STATUS_LOG_FILE, 124 help='Log file location (default: {})'.format( 125 DEFAULT_STATUS_LOG_FILE 126 )) 127 parser.add_argument('--psa-constant-names', metavar='PROGRAM', 128 default=DEFAULT_PSA_CONSTANT_NAMES, 129 help='Path to psa_constant_names (default: {})'.format( 130 DEFAULT_PSA_CONSTANT_NAMES 131 )) 132 parser.add_argument('--use-existing-log', '-e', 133 action='store_true', 134 help='Don\'t regenerate the log file if it exists') 135 options = parser.parse_args() 136 data = collect_status_logs(options) 137 data.report() 138 139if __name__ == '__main__': 140 main() 141