1#!/bin/sh 2 3# basic-build-tests.sh 4# 5# Copyright The Mbed TLS Contributors 6# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 7# 8# This file is provided under the Apache License 2.0, or the 9# GNU General Public License v2.0 or later. 10# 11# ********** 12# Apache License 2.0: 13# 14# Licensed under the Apache License, Version 2.0 (the "License"); you may 15# not use this file except in compliance with the License. 16# You may obtain a copy of the License at 17# 18# http://www.apache.org/licenses/LICENSE-2.0 19# 20# Unless required by applicable law or agreed to in writing, software 21# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 22# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23# See the License for the specific language governing permissions and 24# limitations under the License. 25# 26# ********** 27# 28# ********** 29# GNU General Public License v2.0 or later: 30# 31# This program is free software; you can redistribute it and/or modify 32# it under the terms of the GNU General Public License as published by 33# the Free Software Foundation; either version 2 of the License, or 34# (at your option) any later version. 35# 36# This program is distributed in the hope that it will be useful, 37# but WITHOUT ANY WARRANTY; without even the implied warranty of 38# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 39# GNU General Public License for more details. 40# 41# You should have received a copy of the GNU General Public License along 42# with this program; if not, write to the Free Software Foundation, Inc., 43# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 44# 45# ********** 46# 47# Purpose 48# 49# Executes the basic test suites, captures the results, and generates a simple 50# test report and code coverage report. 51# 52# The tests include: 53# * Unit tests - executed using tests/scripts/run-test-suite.pl 54# * Self-tests - executed using the test suites above 55# * System tests - executed using tests/ssl-opt.sh 56# * Interoperability tests - executed using tests/compat.sh 57# 58# The tests focus on functionality and do not consider performance. 59# 60# Note the tests self-adapt due to configurations in include/mbedtls/config.h 61# which can lead to some tests being skipped, and can cause the number of 62# available tests to fluctuate. 63# 64# This script has been written to be generic and should work on any shell. 65# 66# Usage: basic-build-tests.sh 67# 68 69# Abort on errors (and uninitiliased variables) 70set -eu 71 72if [ -d library -a -d include -a -d tests ]; then :; else 73 echo "Must be run from mbed TLS root" >&2 74 exit 1 75fi 76 77: ${OPENSSL:="openssl"} 78: ${OPENSSL_LEGACY:="$OPENSSL"} 79: ${GNUTLS_CLI:="gnutls-cli"} 80: ${GNUTLS_SERV:="gnutls-serv"} 81: ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"} 82: ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"} 83 84# Used to make ssl-opt.sh deterministic. 85# 86# See also RELEASE_SEED in all.sh. Debugging is easier if both values are kept 87# in sync. If you change the value here because it breaks some tests, you'll 88# definitely want to change it in all.sh as well. 89: ${SEED:=1} 90export SEED 91 92# To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh 93# we just export the variables they require 94export OPENSSL_CMD="$OPENSSL" 95export GNUTLS_CLI="$GNUTLS_CLI" 96export GNUTLS_SERV="$GNUTLS_SERV" 97 98CONFIG_H='include/mbedtls/config.h' 99CONFIG_BAK="$CONFIG_H.bak" 100 101# Step 0 - print build environment info 102OPENSSL="$OPENSSL" \ 103 OPENSSL_LEGACY="$OPENSSL_LEGACY" \ 104 GNUTLS_CLI="$GNUTLS_CLI" \ 105 GNUTLS_SERV="$GNUTLS_SERV" \ 106 GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \ 107 GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" \ 108 scripts/output_env.sh 109echo 110 111# Step 1 - Make and instrumented build for code coverage 112export CFLAGS=' --coverage -g3 -O0 ' 113make clean 114cp "$CONFIG_H" "$CONFIG_BAK" 115scripts/config.pl full 116make -j 117 118 119# Step 2 - Execute the tests 120TEST_OUTPUT=out_${PPID} 121cd tests 122 123if [ ! -f "seedfile" ]; then 124 dd if=/dev/urandom of="seedfile" bs=64 count=1 125fi 126 127# Step 2a - Unit Tests (keep going even if some tests fail) 128perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT 129echo 130 131# Step 2b - System Tests (keep going even if some tests fail) 132sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT 133echo 134 135# Step 2c - Compatibility tests (keep going even if some tests fail) 136sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' | \ 137 tee compat-test-$TEST_OUTPUT 138OPENSSL_CMD="$OPENSSL_LEGACY" \ 139 sh compat.sh -m 'ssl3' |tee -a compat-test-$TEST_OUTPUT 140OPENSSL_CMD="$OPENSSL_LEGACY" \ 141 GNUTLS_CLI="$GNUTLS_LEGACY_CLI" \ 142 GNUTLS_SERV="$GNUTLS_LEGACY_SERV" \ 143 sh compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR' | \ 144 tee -a compat-test-$TEST_OUTPUT 145OPENSSL_CMD="$OPENSSL_NEXT" \ 146 sh compat.sh -e '^$' -f 'ARIA\|CHACHA' | \ 147 tee -a compat-test-$TEST_OUTPUT 148echo 149 150# Step 3 - Process the coverage report 151cd .. 152{ 153 make lcov 154 echo SUCCESS 155} | tee tests/cov-$TEST_OUTPUT 156 157if [ "$(tail -n1 tests/cov-$TEST_OUTPUT)" != "SUCCESS" ]; then 158 echo >&2 "Fatal: 'make lcov' failed" 159 exit 2 160fi 161 162 163# Step 4 - Summarise the test report 164echo 165echo "=========================================================================" 166echo "Test Report Summary" 167echo 168 169cd tests 170 171# Step 4a - Unit tests 172echo "Unit tests - tests/scripts/run-test-suites.pl" 173 174PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ') 175SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ') 176TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ') 177FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ') 178 179echo "No test suites : $TOTAL_SUITES" 180echo "Passed : $PASSED_TESTS" 181echo "Failed : $FAILED_TESTS" 182echo "Skipped : $SKIPPED_TESTS" 183echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))" 184echo "Total avail tests : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))" 185echo 186 187TOTAL_PASS=$PASSED_TESTS 188TOTAL_FAIL=$FAILED_TESTS 189TOTAL_SKIP=$SKIPPED_TESTS 190TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS)) 191TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS)) 192 193# Step 4b - TLS Options tests 194echo "TLS Options tests - tests/ssl-opt.sh" 195 196PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') 197SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') 198TOTAL_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p') 199FAILED_TESTS=$(($TOTAL_TESTS - $PASSED_TESTS)) 200 201echo "Passed : $PASSED_TESTS" 202echo "Failed : $FAILED_TESTS" 203echo "Skipped : $SKIPPED_TESTS" 204echo "Total exec'd tests : $TOTAL_TESTS" 205echo "Total avail tests : $(($TOTAL_TESTS + $SKIPPED_TESTS))" 206echo 207 208TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 209TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 210TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 211TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS)) 212TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS)) 213 214 215# Step 4c - System Compatibility tests 216echo "System/Compatibility tests - tests/compat.sh" 217 218PASSED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 219SKIPPED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 220EXED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 221FAILED_TESTS=$(($EXED_TESTS - $PASSED_TESTS)) 222 223echo "Passed : $PASSED_TESTS" 224echo "Failed : $FAILED_TESTS" 225echo "Skipped : $SKIPPED_TESTS" 226echo "Total exec'd tests : $EXED_TESTS" 227echo "Total avail tests : $(($EXED_TESTS + $SKIPPED_TESTS))" 228echo 229 230TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 231TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 232TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 233TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS)) 234TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS)) 235 236 237# Step 4d - Grand totals 238echo "-------------------------------------------------------------------------" 239echo "Total tests" 240 241echo "Total Passed : $TOTAL_PASS" 242echo "Total Failed : $TOTAL_FAIL" 243echo "Total Skipped : $TOTAL_SKIP" 244echo "Total exec'd tests : $TOTAL_EXED" 245echo "Total avail tests : $TOTAL_AVAIL" 246echo 247 248 249# Step 4e - Coverage 250echo "Coverage" 251 252LINES_TESTED=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* lines)/\1/p') 253LINES_TOTAL=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) lines)/\1/p') 254FUNCS_TESTED=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* functions)$/\1/p') 255FUNCS_TOTAL=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) functions)$/\1/p') 256BRANCHES_TESTED=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ branches...: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* branches)$/\1/p') 257BRANCHES_TOTAL=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ branches...: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) branches)$/\1/p') 258 259LINES_PERCENT=$((1000*$LINES_TESTED/$LINES_TOTAL)) 260LINES_PERCENT="$(($LINES_PERCENT/10)).$(($LINES_PERCENT-($LINES_PERCENT/10)*10))" 261 262FUNCS_PERCENT=$((1000*$FUNCS_TESTED/$FUNCS_TOTAL)) 263FUNCS_PERCENT="$(($FUNCS_PERCENT/10)).$(($FUNCS_PERCENT-($FUNCS_PERCENT/10)*10))" 264 265BRANCHES_PERCENT=$((1000*$BRANCHES_TESTED/$BRANCHES_TOTAL)) 266BRANCHES_PERCENT="$(($BRANCHES_PERCENT/10)).$(($BRANCHES_PERCENT-($BRANCHES_PERCENT/10)*10))" 267 268echo "Lines Tested : $LINES_TESTED of $LINES_TOTAL $LINES_PERCENT%" 269echo "Functions Tested : $FUNCS_TESTED of $FUNCS_TOTAL $FUNCS_PERCENT%" 270echo "Branches Tested : $BRANCHES_TESTED of $BRANCHES_TOTAL $BRANCHES_PERCENT%" 271echo 272 273rm unit-test-$TEST_OUTPUT 274rm sys-test-$TEST_OUTPUT 275rm compat-test-$TEST_OUTPUT 276rm cov-$TEST_OUTPUT 277 278cd .. 279 280make clean 281 282if [ -f "$CONFIG_BAK" ]; then 283 mv "$CONFIG_BAK" "$CONFIG_H" 284fi 285 286if [ $TOTAL_FAIL -ne 0 ]; then 287 exit 1 288fi 289