Lines Matching +full:check +full:- +full:test +full:- +full:suites
3 """Analyze the test outcomes from a full CI run.
71 outcome_file: str) -> None:
75 results.new_section("Test {} and {}", ref_component, driver_component)
77 shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \
80 ret_val = subprocess.run(shell_command.split(), check=False).returncode
86 allow_list: typing.List[str], full_coverage: bool) -> None:
87 """Check that all available test cases are executed at least once."""
96 results.error('Test case not executed: {}', suite_case)
98 results.warning('Test case not executed: {}', suite_case)
100 # Test Case should be removed from the allow list.
102 results.error('Allow listed test case was executed: {}', suite_case)
104 results.warning('Allow listed test case was executed: {}', suite_case)
106 def name_matches_pattern(name: str, str_or_re) -> bool:
107 """Check if name matches a pattern, that may be a string or regex.
108 - If the pattern is a string, name must be equal to match.
109 - If the pattern is a regex, name must fully match.
120 ignored_suites: typing.List[str], ignored_tests=None) -> None:
121 """Check that all tests passing in the reference component are also
124 - full test suites provided in ignored_suites list
125 - only some specific test inside a test suite, for which the corresponding
136 results.error("no passing test in reference component: bad outcome file?")
140 # suite_case is like "test_suite_foo.bar;Description of test case"
142 test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name
144 # Immediately skip fully-ignored test suites
148 # For ignored test cases inside test suites, just remember and:
158 results.error("PASS -> SKIP/FAIL: {}", suite_case)
162 def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None:
167 def read_outcome_file(outcome_file: str) -> Outcomes:
171 with open(outcome_file, 'r', encoding='utf-8') as input_file:
174 # Note that `component` is not unique. If a test case passes on Linux
187 def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None:
192 def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None:
219 # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
220 # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
222 # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
238 'Check mbedtls_calloc overallocation',
249 # These suites require legacy hash support, which is disabled
262 # test component.
269 # test component.
276 'Check mbedtls_calloc overallocation',
288 # low-level (block/stream) cipher modules
305 'Check mbedtls_calloc overallocation',
315 'Check for MBEDTLS_AES_C when already present',
330 # pylint: disable=line-too-long
353 'Check mbedtls_calloc overallocation',
355 # This test wants a legacy function that takes f_rng, p_rng
356 # arguments, and uses legacy ECDSA for that. The test is
361 # In the accelerated test ECP_C is not set (only ECP_LIGHT is)
365 re.compile(r'ECP check public-private .*'),
370 re.compile(r'ECP test vectors .*'),
374 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
393 'Check mbedtls_calloc overallocation',
411 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
432 'Check mbedtls_calloc overallocation',
454 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
475 'Check mbedtls_calloc overallocation',
497 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
513 'Check mbedtls_calloc overallocation',
535 'Check mbedtls_calloc overallocation',
560 'Check mbedtls_calloc overallocation',
606 re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'),
611 re.compile(r'PBES2 .* AES-.*')
617 re.compile('Parse RSA Key.*(password|AES-).*'),
633 'Check for MBEDTLS_AES_C when already present',
639 'Check mbedtls_calloc overallocation',
657 'comma/space-separated list of tasks. ')
658 parser.add_argument('--list', action='store_true',
660 parser.add_argument('--require-full-coverage', action='store_true',
662 "test cases to be executed and issue an error "
693 …task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable
714 except Exception: # pylint: disable=broad-except