• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2"""Run the PSA Crypto API compliance test suite.
3Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF,
4then compile and run the test suite. The clone is stored at <Mbed TLS root>/psa-arch-tests.
5Known defects in either the test suite or mbedtls - identified by their test number - are ignored,
6while unexpected failures AND successes are reported as errors,
7to help keep the list of known defects as up to date as possible.
8"""
9
10# Copyright The Mbed TLS Contributors
11# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
12
13import os
14import re
15import shutil
16import subprocess
17import sys
18
19# PSA Compliance tests we expect to fail due to known defects in Mbed TLS (or the test suite)
20# The test numbers correspond to the numbers used by the console output of the test suite.
21# Test number 2xx corresponds to the files in the folder
22# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx
23EXPECTED_FAILURES = {
24    # Multipart AEAD is not supported in Mbed TLS 2.x.
25    252, 253, 254, 255, 256, 257, 258, 259, 260, 261,
26
27    # psa_hash_suspend() and psa_hash_resume() are not supported.
28    # - Tracked in issue #3274
29    262, 263
30}
31
32# We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches
33# that allow it to build with Mbed TLS 2, and fixes a couple of issues in the compliance test suite.
34# These fixes allow the tests numbered 216, 248 and 249 to complete successfully.
35#
36# Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag.
37# - Tracked in issue #5145
38#
39# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-2
40PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git'
41PSA_ARCH_TESTS_REF = 'fix-5735-2.28'
42
43#pylint: disable=too-many-branches,too-many-statements
44def main():
45    mbedtls_dir = os.getcwd()
46
47    if not os.path.exists('library/libmbedcrypto.a'):
48        subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a'])
49
50    psa_arch_tests_dir = 'psa-arch-tests'
51    os.makedirs(psa_arch_tests_dir, exist_ok=True)
52    try:
53        os.chdir(psa_arch_tests_dir)
54
55        # Reuse existing local clone
56        subprocess.check_call(['git', 'init'])
57        subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF])
58        subprocess.check_call(['git', 'checkout', 'FETCH_HEAD'])
59
60        build_dir = 'api-tests/build'
61        try:
62            shutil.rmtree(build_dir)
63        except FileNotFoundError:
64            pass
65        os.mkdir(build_dir)
66        os.chdir(build_dir)
67
68        #pylint: disable=bad-continuation
69        subprocess.check_call([
70            'cmake', '..',
71                     '-GUnix Makefiles',
72                     '-DTARGET=tgt_dev_apis_stdc',
73                     '-DTOOLCHAIN=HOST_GCC',
74                     '-DSUITE=CRYPTO',
75                     '-DMISSING_CRYPTO_1_0=1',
76                     '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir),
77                     '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir)
78        ])
79        subprocess.check_call(['cmake', '--build', '.'])
80
81        proc = subprocess.Popen(['./psa-arch-tests-crypto'],
82                                bufsize=1, stdout=subprocess.PIPE, universal_newlines=True)
83
84        test_re = re.compile(
85            '^TEST: (?P<test_num>[0-9]*)|'
86            '^TEST RESULT: (?P<test_result>FAILED|PASSED)'
87        )
88        test = -1
89        unexpected_successes = set(EXPECTED_FAILURES)
90        expected_failures = []
91        unexpected_failures = []
92        for line in proc.stdout:
93            print(line, end='')
94            match = test_re.match(line)
95            if match is not None:
96                groupdict = match.groupdict()
97                test_num = groupdict['test_num']
98                if test_num is not None:
99                    test = int(test_num)
100                elif groupdict['test_result'] == 'FAILED':
101                    try:
102                        unexpected_successes.remove(test)
103                        expected_failures.append(test)
104                        print('Expected failure, ignoring')
105                    except KeyError:
106                        unexpected_failures.append(test)
107                        print('ERROR: Unexpected failure')
108                elif test in unexpected_successes:
109                    print('ERROR: Unexpected success')
110        proc.wait()
111
112        print()
113        print('***** test_psa_compliance.py report ******')
114        print()
115        print('Expected failures:', ', '.join(str(i) for i in expected_failures))
116        print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures))
117        print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes)))
118        print()
119        if unexpected_successes or unexpected_failures:
120            if unexpected_successes:
121                print('Unexpected successes encountered.')
122                print('Please remove the corresponding tests from '
123                      'EXPECTED_FAILURES in tests/scripts/compliance_test.py')
124                print()
125            print('FAILED')
126            return 1
127        else:
128            print('SUCCESS')
129            return 0
130    finally:
131        os.chdir(mbedtls_dir)
132
133if __name__ == '__main__':
134    sys.exit(main())
135