1#!/usr/bin/env python3 2"""Run the PSA Cryto API compliance test suite. 3Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, 4then complie and run the test suite. The clone is stored at <Mbed TLS root>/psa-arch-tests. 5Known defects in either the test suite or mbedtls - identified by their test number - are ignored, 6while unexpected failures AND successes are reported as errors, 7to help keep the list of known defects as up to date as possible. 8""" 9 10# Copyright The Mbed TLS Contributors 11# SPDX-License-Identifier: Apache-2.0 12# 13# Licensed under the Apache License, Version 2.0 (the "License"); you may 14# not use this file except in compliance with the License. 15# You may obtain a copy of the License at 16# 17# http://www.apache.org/licenses/LICENSE-2.0 18# 19# Unless required by applicable law or agreed to in writing, software 20# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 21# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 22# See the License for the specific language governing permissions and 23# limitations under the License. 24 25import os 26import re 27import shutil 28import subprocess 29import sys 30 31# PSA Compliance tests we expect to fail due to known defects in Mbed TLS (or the test suite) 32# The test numbers correspond to the numbers used by the console output of the test suite. 33# Test number 2xx corresponds to the files in the folder 34# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx 35EXPECTED_FAILURES = { 36 # Multipart AEAD is not supported in Mbed TLS 2.x. 37 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 38 39 # psa_hash_suspend() and psa_hash_resume() are not supported. 40 # - Tracked in issue #3274 41 262, 263 42} 43 44# We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches 45# that allow it to build with Mbed TLS 2, and fixes a couple of issues in the compliance test suite. 46# These fixes allow the tests numbered 216, 248 and 249 to complete successfully. 47# 48# Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag. 49# - Tracked in issue #5145 50# 51# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-2 52PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git' 53PSA_ARCH_TESTS_REF = 'fixes-for-mbedtls-2' 54 55#pylint: disable=too-many-branches,too-many-statements 56def main(): 57 mbedtls_dir = os.getcwd() 58 59 if not os.path.exists('library/libmbedcrypto.a'): 60 subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a']) 61 62 psa_arch_tests_dir = 'psa-arch-tests' 63 os.makedirs(psa_arch_tests_dir, exist_ok=True) 64 try: 65 os.chdir(psa_arch_tests_dir) 66 67 # Reuse existing local clone 68 subprocess.check_call(['git', 'init']) 69 subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) 70 subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) 71 72 build_dir = 'api-tests/build' 73 try: 74 shutil.rmtree(build_dir) 75 except FileNotFoundError: 76 pass 77 os.mkdir(build_dir) 78 os.chdir(build_dir) 79 80 #pylint: disable=bad-continuation 81 subprocess.check_call([ 82 'cmake', '..', 83 '-GUnix Makefiles', 84 '-DTARGET=tgt_dev_apis_stdc', 85 '-DTOOLCHAIN=HOST_GCC', 86 '-DSUITE=CRYPTO', 87 '-DMISSING_CRYPTO_1_0=1', 88 '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), 89 '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) 90 ]) 91 subprocess.check_call(['cmake', '--build', '.']) 92 93 proc = subprocess.Popen(['./psa-arch-tests-crypto'], 94 bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) 95 96 test_re = re.compile( 97 '^TEST: (?P<test_num>[0-9]*)|' 98 '^TEST RESULT: (?P<test_result>FAILED|PASSED)' 99 ) 100 test = -1 101 unexpected_successes = set(EXPECTED_FAILURES) 102 expected_failures = [] 103 unexpected_failures = [] 104 for line in proc.stdout: 105 print(line, end='') 106 match = test_re.match(line) 107 if match is not None: 108 groupdict = match.groupdict() 109 test_num = groupdict['test_num'] 110 if test_num is not None: 111 test = int(test_num) 112 elif groupdict['test_result'] == 'FAILED': 113 try: 114 unexpected_successes.remove(test) 115 expected_failures.append(test) 116 print('Expected failure, ignoring') 117 except KeyError: 118 unexpected_failures.append(test) 119 print('ERROR: Unexpected failure') 120 elif test in unexpected_successes: 121 print('ERROR: Unexpected success') 122 proc.wait() 123 124 print() 125 print('***** test_psa_compliance.py report ******') 126 print() 127 print('Expected failures:', ', '.join(str(i) for i in expected_failures)) 128 print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) 129 print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) 130 print() 131 if unexpected_successes or unexpected_failures: 132 if unexpected_successes: 133 print('Unexpected successes encountered.') 134 print('Please remove the corresponding tests from ' 135 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') 136 print() 137 print('FAILED') 138 return 1 139 else: 140 print('SUCCESS') 141 return 0 142 finally: 143 os.chdir(mbedtls_dir) 144 145if __name__ == '__main__': 146 sys.exit(main()) 147