• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/bin/sh
2
3# This script runs tests before and after a PR and analyzes the results in
4# order to highlight any difference in the set of tests skipped.
5#
6# It can be used to check the first testing criterion mentioned in strategy.md,
7# end of section "Supporting builds with drivers without the software
8# implementation", namely: the sets of tests skipped in the default config and
9# the full config must be the same before and after the PR.
10#
11# USAGE:
12# - First, commit any uncommited changes. (Also, see warning below.)
13# - Then launch --> [SKIP_SSL_OPT=1] docs/architecture/psa-migration/outcome-analysis.sh
14#     - SKIP_SSL_OPT=1 can optionally be set to skip ssl-opt.sh tests
15#
16# WARNING: this script checks out a commit other than the head of the current
17# branch; it checks out the current branch again when running successfully,
18# but while the script is running, or if it terminates early in error, you
19# should be aware that you might be at a different commit than expected.
20#
21# NOTE: you can comment out parts that don't need to be re-done when
22# re-running this script (for example "get numbers before this PR").
23
24set -eu
25
26: ${SKIP_SSL_OPT:=0}
27
28cleanup() {
29    make clean
30    git checkout -- include/mbedtls/mbedtls_config.h include/psa/crypto_config.h
31}
32
33record() {
34    export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-$1.csv"
35    rm -f $MBEDTLS_TEST_OUTCOME_FILE
36
37    make check
38
39    if [ $SKIP_SSL_OPT -eq 0 ]; then
40        make -C programs ssl/ssl_server2 ssl/ssl_client2 \
41            test/udp_proxy test/query_compile_time_config
42        tests/ssl-opt.sh
43    fi
44}
45
46# save current HEAD
47HEAD=$(git branch --show-current)
48
49# get the numbers before this PR for default and full
50cleanup
51git checkout $(git merge-base HEAD development)
52
53record "before-default"
54
55cleanup
56
57scripts/config.py full
58record "before-full"
59
60# get the numbers now for default and full
61cleanup
62git checkout $HEAD
63
64record "after-default"
65
66cleanup
67
68scripts/config.py full
69record "after-full"
70
71cleanup
72
73# analysis
74
75populate_suites () {
76    SUITES=''
77    make generated_files >/dev/null
78    data_files=$(cd tests/suites && echo *.data)
79    for data in $data_files; do
80        suite=${data%.data}
81        SUITES="$SUITES $suite"
82    done
83    make neat
84
85    if [ $SKIP_SSL_OPT -eq 0 ]; then
86        SUITES="$SUITES ssl-opt"
87        extra_files=$(cd tests/opt-testcases && echo *.sh)
88        for extra in $extra_files; do
89            suite=${extra%.sh}
90            SUITES="$SUITES $suite"
91        done
92    fi
93}
94
95compare_suite () {
96    ref="outcome-$1.csv"
97    new="outcome-$2.csv"
98    suite="$3"
99
100    pattern_suite=";$suite;"
101    total=$(grep -c "$pattern_suite" "$ref")
102    sed_cmd="s/^.*$pattern_suite\(.*\);SKIP.*/\1/p"
103    sed -n "$sed_cmd" "$ref" > skipped-ref
104    sed -n "$sed_cmd" "$new" > skipped-new
105    nb_ref=$(wc -l <skipped-ref)
106    nb_new=$(wc -l <skipped-new)
107
108    name=${suite#test_suite_}
109    printf "%40s: total %4d; skipped %4d -> %4d\n" \
110            $name       $total       $nb_ref $nb_new
111    if diff skipped-ref skipped-new | grep '^> '; then
112        ret=1
113    else
114        ret=0
115    fi
116    rm skipped-ref skipped-new
117    return $ret
118}
119
120compare_builds () {
121    printf "\n*** Comparing $1 -> $2 ***\n"
122    failed=''
123    for suite in $SUITES; do
124        if compare_suite "$1" "$2" "$suite"; then :; else
125            failed="$failed $suite"
126        fi
127    done
128    if [ -z "$failed" ]; then
129        printf "No coverage gap found.\n"
130    else
131        printf "Suites with less coverage:%s\n" "$failed"
132    fi
133}
134
135populate_suites
136compare_builds before-default after-default
137compare_builds before-full after-full
138