1#!/bin/sh 2 3# basic-build-test.sh 4# 5# Copyright The Mbed TLS Contributors 6# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 7# 8# Purpose 9# 10# Executes the basic test suites, captures the results, and generates a simple 11# test report and code coverage report. 12# 13# The tests include: 14# * Unit tests - executed using tests/scripts/run-test-suite.pl 15# * Self-tests - executed using the test suites above 16# * System tests - executed using tests/ssl-opt.sh 17# * Interoperability tests - executed using tests/compat.sh 18# 19# The tests focus on functionality and do not consider performance. 20# 21# Note the tests self-adapt due to configurations in include/mbedtls/config.h 22# which can lead to some tests being skipped, and can cause the number of 23# available tests to fluctuate. 24# 25# This script has been written to be generic and should work on any shell. 26# 27# Usage: basic-build-test.sh 28# 29 30# Abort on errors (and uninitiliased variables) 31set -eu 32 33if [ -d library -a -d include -a -d tests ]; then :; else 34 echo "Must be run from Mbed TLS root" >&2 35 exit 1 36fi 37 38: ${OPENSSL:="openssl"} 39: ${OPENSSL_LEGACY:="$OPENSSL"} 40: ${GNUTLS_CLI:="gnutls-cli"} 41: ${GNUTLS_SERV:="gnutls-serv"} 42: ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"} 43: ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"} 44 45# Used to make ssl-opt.sh deterministic. 46# 47# See also RELEASE_SEED in all.sh. Debugging is easier if both values are kept 48# in sync. If you change the value here because it breaks some tests, you'll 49# definitely want to change it in all.sh as well. 50: ${SEED:=1} 51export SEED 52 53# if MAKEFLAGS is not set add the -j option to speed up invocations of make 54if [ -z "${MAKEFLAGS+set}" ]; then 55 export MAKEFLAGS="-j" 56fi 57 58# To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh 59# we just export the variables they require 60export OPENSSL="$OPENSSL" 61export GNUTLS_CLI="$GNUTLS_CLI" 62export GNUTLS_SERV="$GNUTLS_SERV" 63 64CONFIG_H='include/mbedtls/config.h' 65CONFIG_BAK="$CONFIG_H.bak" 66 67# Step 0 - print build environment info 68OPENSSL="$OPENSSL" \ 69 OPENSSL_LEGACY="$OPENSSL_LEGACY" \ 70 GNUTLS_CLI="$GNUTLS_CLI" \ 71 GNUTLS_SERV="$GNUTLS_SERV" \ 72 GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \ 73 GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" \ 74 scripts/output_env.sh 75echo 76 77# Step 1 - Make and instrumented build for code coverage 78export CFLAGS=' --coverage -g3 -O0 ' 79export LDFLAGS=' --coverage' 80make clean 81cp "$CONFIG_H" "$CONFIG_BAK" 82scripts/config.py full 83make 84 85 86# Step 2 - Execute the tests 87TEST_OUTPUT=out_${PPID} 88cd tests 89if [ ! -f "seedfile" ]; then 90 dd if=/dev/urandom of="seedfile" bs=64 count=1 91fi 92echo 93 94# Step 2a - Unit Tests (keep going even if some tests fail) 95echo '################ Unit tests ################' 96perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT 97echo '^^^^^^^^^^^^^^^^ Unit tests ^^^^^^^^^^^^^^^^' 98echo 99 100# Step 2b - System Tests (keep going even if some tests fail) 101echo 102echo '################ ssl-opt.sh ################' 103echo "ssl-opt.sh will use SEED=$SEED for udp_proxy" 104sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT 105echo '^^^^^^^^^^^^^^^^ ssl-opt.sh ^^^^^^^^^^^^^^^^' 106echo 107 108# Step 2c - Compatibility tests (keep going even if some tests fail) 109echo '################ compat.sh ################' 110{ 111 echo '#### compat.sh: Default versions' 112 sh compat.sh -m 'tls1 tls1_1 tls12 dtls1 dtls12' 113 echo 114 115 echo '#### compat.sh: legacy (SSLv3)' 116 OPENSSL="$OPENSSL_LEGACY" sh compat.sh -m 'ssl3' 117 echo 118 119 echo '#### compat.sh: legacy (null, DES, RC4)' 120 OPENSSL="$OPENSSL_LEGACY" \ 121 GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" \ 122 sh compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR' 123 echo 124 125 echo '#### compat.sh: next (ARIA, ChaCha)' 126 OPENSSL="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA' 127 echo 128} | tee compat-test-$TEST_OUTPUT 129echo '^^^^^^^^^^^^^^^^ compat.sh ^^^^^^^^^^^^^^^^' 130echo 131 132# Step 3 - Process the coverage report 133cd .. 134{ 135 make lcov 136 echo SUCCESS 137} | tee tests/cov-$TEST_OUTPUT 138 139if [ "$(tail -n1 tests/cov-$TEST_OUTPUT)" != "SUCCESS" ]; then 140 echo >&2 "Fatal: 'make lcov' failed" 141 exit 2 142fi 143 144 145# Step 4 - Summarise the test report 146echo 147echo "=========================================================================" 148echo "Test Report Summary" 149echo 150 151# A failure of the left-hand side of a pipe is ignored (this is a limitation 152# of sh). We'll use the presence of this file as a marker that the generation 153# of the report succeeded. 154rm -f "tests/basic-build-test-$$.ok" 155 156{ 157 158 cd tests 159 160 # Step 4a - Unit tests 161 echo "Unit tests - tests/scripts/run-test-suites.pl" 162 163 PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ') 164 SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ') 165 TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ') 166 FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ') 167 168 echo "No test suites : $TOTAL_SUITES" 169 echo "Passed : $PASSED_TESTS" 170 echo "Failed : $FAILED_TESTS" 171 echo "Skipped : $SKIPPED_TESTS" 172 echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))" 173 echo "Total avail tests : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))" 174 echo 175 176 TOTAL_PASS=$PASSED_TESTS 177 TOTAL_FAIL=$FAILED_TESTS 178 TOTAL_SKIP=$SKIPPED_TESTS 179 TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS)) 180 TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS)) 181 182 # Step 4b - TLS Options tests 183 echo "TLS Options tests - tests/ssl-opt.sh" 184 185 PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') 186 SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') 187 TOTAL_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p') 188 FAILED_TESTS=$(($TOTAL_TESTS - $PASSED_TESTS)) 189 190 echo "Passed : $PASSED_TESTS" 191 echo "Failed : $FAILED_TESTS" 192 echo "Skipped : $SKIPPED_TESTS" 193 echo "Total exec'd tests : $TOTAL_TESTS" 194 echo "Total avail tests : $(($TOTAL_TESTS + $SKIPPED_TESTS))" 195 echo 196 197 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 198 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 199 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 200 TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS)) 201 TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS)) 202 203 204 # Step 4c - System Compatibility tests 205 echo "System/Compatibility tests - tests/compat.sh" 206 207 PASSED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 208 SKIPPED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 209 EXED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 210 FAILED_TESTS=$(($EXED_TESTS - $PASSED_TESTS)) 211 212 echo "Passed : $PASSED_TESTS" 213 echo "Failed : $FAILED_TESTS" 214 echo "Skipped : $SKIPPED_TESTS" 215 echo "Total exec'd tests : $EXED_TESTS" 216 echo "Total avail tests : $(($EXED_TESTS + $SKIPPED_TESTS))" 217 echo 218 219 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 220 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 221 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 222 TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS)) 223 TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS)) 224 225 226 # Step 4d - Grand totals 227 echo "-------------------------------------------------------------------------" 228 echo "Total tests" 229 230 echo "Total Passed : $TOTAL_PASS" 231 echo "Total Failed : $TOTAL_FAIL" 232 echo "Total Skipped : $TOTAL_SKIP" 233 echo "Total exec'd tests : $TOTAL_EXED" 234 echo "Total avail tests : $TOTAL_AVAIL" 235 echo 236 237 238 # Step 4e - Coverage report 239 echo "Coverage statistics:" 240 sed -n '1,/^Overall coverage/d; /%/p' cov-$TEST_OUTPUT 241 echo 242 243 rm unit-test-$TEST_OUTPUT 244 rm sys-test-$TEST_OUTPUT 245 rm compat-test-$TEST_OUTPUT 246 rm cov-$TEST_OUTPUT 247 248 # Mark the report generation as having succeeded. This must be the 249 # last thing in the report generation. 250 touch "basic-build-test-$$.ok" 251} | tee coverage-summary.txt 252 253make clean 254 255if [ -f "$CONFIG_BAK" ]; then 256 mv "$CONFIG_BAK" "$CONFIG_H" 257fi 258 259# The file must exist, otherwise it means something went wrong while generating 260# the coverage report. If something did go wrong, rm will complain so this 261# script will exit with a failure status. 262rm "tests/basic-build-test-$$.ok" 263