1#!/usr/bin/env perl 2 3# run-test-suites.pl 4# 5# Copyright The Mbed TLS Contributors 6# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 7# 8# This file is provided under the Apache License 2.0, or the 9# GNU General Public License v2.0 or later. 10# 11# ********** 12# Apache License 2.0: 13# 14# Licensed under the Apache License, Version 2.0 (the "License"); you may 15# not use this file except in compliance with the License. 16# You may obtain a copy of the License at 17# 18# http://www.apache.org/licenses/LICENSE-2.0 19# 20# Unless required by applicable law or agreed to in writing, software 21# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 22# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23# See the License for the specific language governing permissions and 24# limitations under the License. 25# 26# ********** 27# 28# ********** 29# GNU General Public License v2.0 or later: 30# 31# This program is free software; you can redistribute it and/or modify 32# it under the terms of the GNU General Public License as published by 33# the Free Software Foundation; either version 2 of the License, or 34# (at your option) any later version. 35# 36# This program is distributed in the hope that it will be useful, 37# but WITHOUT ANY WARRANTY; without even the implied warranty of 38# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 39# GNU General Public License for more details. 40# 41# You should have received a copy of the GNU General Public License along 42# with this program; if not, write to the Free Software Foundation, Inc., 43# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 44# 45# ********** 46 47=head1 SYNOPSIS 48 49Execute all the test suites and print a summary of the results. 50 51 run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]] 52 53Options: 54 55 -v|--verbose Print detailed failure information. 56 -v 2|--verbose=2 Print detailed failure information and summary messages. 57 -v 3|--verbose=3 Print detailed information about every test case. 58 --skip=SUITE[,SUITE...] 59 Skip the specified SUITE(s). This option can be used 60 multiple times. 61 62=cut 63 64use warnings; 65use strict; 66 67use utf8; 68use open qw(:std utf8); 69 70use Getopt::Long qw(:config auto_help gnu_compat); 71use Pod::Usage; 72 73my $verbose = 0; 74my @skip_patterns = (); 75GetOptions( 76 'skip=s' => \@skip_patterns, 77 'verbose|v:1' => \$verbose, 78 ) or die; 79 80# All test suites = executable files, excluding source files, debug 81# and profiling information, etc. We can't just grep {! /\./} because 82# some of our test cases' base names contain a dot. 83my @suites = grep { -x $_ || /\.exe$/ } glob 'test_suite_*'; 84@suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites; 85die "$0: no test suite found\n" unless @suites; 86 87# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar" 88# but not "test_suite_foobar". 89my $skip_re = 90 ( '\Atest_suite_(' . 91 join('|', map { 92 s/[ ,;]/|/g; # allow any of " ,;|" as separators 93 s/\./\./g; # "." in the input means ".", not "any character" 94 $_ 95 } @skip_patterns) . 96 ')(\z|\.)' ); 97 98# in case test suites are linked dynamically 99$ENV{'LD_LIBRARY_PATH'} = '../library'; 100$ENV{'DYLD_LIBRARY_PATH'} = '../library'; 101 102my $prefix = $^O eq "MSWin32" ? '' : './'; 103 104my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed, 105 $suite_cases_failed, $suite_cases_skipped, $total_cases_passed, 106 $total_cases_failed, $total_cases_skipped ); 107my $suites_skipped = 0; 108 109sub pad_print_center { 110 my( $width, $padchar, $string ) = @_; 111 my $padlen = ( $width - length( $string ) - 2 ) / 2; 112 print $padchar x( $padlen ), " $string ", $padchar x( $padlen ), "\n"; 113} 114 115for my $suite (@suites) 116{ 117 print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " "; 118 if( $suite =~ /$skip_re/o ) { 119 print "SKIP\n"; 120 ++$suites_skipped; 121 next; 122 } 123 124 my $command = "$prefix$suite"; 125 if( $verbose ) { 126 $command .= ' -v'; 127 } 128 my $result = `$command`; 129 130 $suite_cases_passed = () = $result =~ /.. PASS/g; 131 $suite_cases_failed = () = $result =~ /.. FAILED/g; 132 $suite_cases_skipped = () = $result =~ /.. ----/g; 133 134 if( $? == 0 ) { 135 print "PASS\n"; 136 if( $verbose > 2 ) { 137 pad_print_center( 72, '-', "Begin $suite" ); 138 print $result; 139 pad_print_center( 72, '-', "End $suite" ); 140 } 141 } else { 142 $failed_suites++; 143 print "FAIL\n"; 144 if( $verbose ) { 145 pad_print_center( 72, '-', "Begin $suite" ); 146 print $result; 147 pad_print_center( 72, '-', "End $suite" ); 148 } 149 } 150 151 my ($passed, $tests, $skipped) = $result =~ /([0-9]*) \/ ([0-9]*) tests.*?([0-9]*) skipped/; 152 $total_tests_run += $tests - $skipped; 153 154 if( $verbose > 1 ) { 155 print "(test cases passed:", $suite_cases_passed, 156 " failed:", $suite_cases_failed, 157 " skipped:", $suite_cases_skipped, 158 " of total:", ($suite_cases_passed + $suite_cases_failed + 159 $suite_cases_skipped), 160 ")\n" 161 } 162 163 $total_cases_passed += $suite_cases_passed; 164 $total_cases_failed += $suite_cases_failed; 165 $total_cases_skipped += $suite_cases_skipped; 166} 167 168print "-" x 72, "\n"; 169print $failed_suites ? "FAILED" : "PASSED"; 170printf( " (%d suites, %d tests run%s)\n", 171 scalar(@suites) - $suites_skipped, 172 $total_tests_run, 173 $suites_skipped ? ", $suites_skipped suites skipped" : "" ); 174 175if( $verbose > 1 ) { 176 print " test cases passed :", $total_cases_passed, "\n"; 177 print " failed :", $total_cases_failed, "\n"; 178 print " skipped :", $total_cases_skipped, "\n"; 179 print " of tests executed :", ( $total_cases_passed + $total_cases_failed ), 180 "\n"; 181 print " of available tests :", 182 ( $total_cases_passed + $total_cases_failed + $total_cases_skipped ), 183 "\n"; 184 if( $suites_skipped != 0 ) { 185 print "Note: $suites_skipped suites were skipped.\n"; 186 } 187} 188 189exit( $failed_suites ? 1 : 0 ); 190 191