1#!/bin/bash 2 3# Copyright JS Foundation and other contributors, http://js.foundation 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17ITERS="$1" 18ENGINE="$2" 19BENCHMARK="$3" 20PRINT_MIN="$4" 21OS=`uname -s | tr [:upper:] [:lower:]` 22 23if [ "$OS" == "darwin" ] 24then 25 time_regexp='s/user[ ]*\([0-9]*\)m\([0-9.]*\)s/\1 \2/g' 26else 27 time_regexp='s/user[ \t]*\([0-9]*\)m\([0-9.]*\)s/\1 \2/g' 28fi 29 30perf_values=$( (( for i in `seq 1 1 $ITERS`; do time $ENGINE "$BENCHMARK"; if [ $? -ne 0 ]; then exit 1; fi; done ) 2>&1 ) | \ 31 grep user | \ 32 sed "$time_regexp" | \ 33 awk '{ print ($1 * 60 + $2); }'; 34 if [ ${PIPESTATUS[0]} -ne 0 ]; then exit 1; fi; ); 35 36if [ "$PRINT_MIN" == "-min" ] 37then 38 perf_values=$( echo "$perf_values" | \ 39 awk "BEGIN { 40 min_v = -1; 41 } 42 { 43 if (min_v == -1 || $1 < min_v) { 44 min_v = $1; 45 } 46 } 47 END { 48 print min_v 49 }" || exit 1; 50 ); 51 calc_status=$? 52else 53 perf_values=$( echo "$perf_values" | \ 54 awk "BEGIN { 55 n = 0 56 } 57 { 58 n++ 59 a[n] = \$1 60 } 61 END { 62 # 63 # Values of 99% quantiles of two-sided t-distribution for given number of degrees of freedom 64 # 65 t_gamma_n_m1 [1] = 63.657 66 t_gamma_n_m1 [2] = 9.9248 67 t_gamma_n_m1 [3] = 5.8409 68 t_gamma_n_m1 [4] = 4.6041 69 t_gamma_n_m1 [5] = 4.0321 70 t_gamma_n_m1 [6] = 3.7074 71 t_gamma_n_m1 [7] = 3.4995 72 t_gamma_n_m1 [8] = 3.3554 73 t_gamma_n_m1 [9] = 3.2498 74 t_gamma_n_m1 [10] = 3.1693 75 t_gamma_n_m1 [11] = 3.1058 76 t_gamma_n_m1 [12] = 3.0545 77 t_gamma_n_m1 [13] = 3.0123 78 t_gamma_n_m1 [14] = 2.9768 79 t_gamma_n_m1 [15] = 2.9467 80 t_gamma_n_m1 [16] = 2.9208 81 t_gamma_n_m1 [17] = 2.8982 82 t_gamma_n_m1 [18] = 2.8784 83 t_gamma_n_m1 [19] = 2.8609 84 t_gamma_n_m1 [20] = 2.8453 85 t_gamma_n_m1 [21] = 2.8314 86 t_gamma_n_m1 [22] = 2.8188 87 t_gamma_n_m1 [23] = 2.8073 88 t_gamma_n_m1 [24] = 2.7969 89 t_gamma_n_m1 [25] = 2.7874 90 t_gamma_n_m1 [26] = 2.7787 91 t_gamma_n_m1 [27] = 2.7707 92 t_gamma_n_m1 [28] = 2.7633 93 t_gamma_n_m1 [29] = 2.7564 94 t_gamma_n_m1 [30] = 2.75 95 t_gamma_n_m1 [31] = 2.744 96 t_gamma_n_m1 [32] = 2.7385 97 t_gamma_n_m1 [33] = 2.7333 98 t_gamma_n_m1 [34] = 2.7284 99 t_gamma_n_m1 [35] = 2.7238 100 t_gamma_n_m1 [36] = 2.7195 101 t_gamma_n_m1 [37] = 2.7154 102 t_gamma_n_m1 [38] = 2.7116 103 t_gamma_n_m1 [39] = 2.7079 104 t_gamma_n_m1 [40] = 2.7045 105 t_gamma_n_m1 [41] = 2.7012 106 t_gamma_n_m1 [42] = 2.6981 107 t_gamma_n_m1 [43] = 2.6951 108 t_gamma_n_m1 [44] = 2.6923 109 t_gamma_n_m1 [45] = 2.6896 110 t_gamma_n_m1 [46] = 2.687 111 t_gamma_n_m1 [47] = 2.6846 112 t_gamma_n_m1 [48] = 2.6822 113 t_gamma_n_m1 [49] = 2.68 114 t_gamma_n_m1 [50] = 2.6778 115 116 # 117 # Sort array of measurements 118 # 119 for (i = 2; i <= n; i++) { 120 j = i 121 k = a [j] 122 while (j > 1 && a [j - 1] > k) { 123 a [j] = a [j - 1] 124 j-- 125 } 126 a [j] = k 127 } 128 129 # 130 # Remove 20% of lowest and 20% of highest values 131 # 132 n_20_percent = int (n / 5) 133 134 for (i = 1; i <= n_20_percent; i++) { 135 delete a[n] 136 n-- 137 } 138 139 for (i = 1; i <= n - n_20_percent; i++) { 140 a[i] = a[i + n_20_percent] 141 } 142 143 n -= n_20_percent 144 145 # 146 # Calculate average 147 # 148 sum = 0 149 for (i = 1; i <= n; i++) { 150 sum += a[i] 151 } 152 153 avg = sum / n 154 155 if (n > 1) { 156 if (n - 1 <= 50) { 157 t_coef = t_gamma_n_m1 [n - 1] 158 } else { 159 # For greater degrees of freedom, values of corresponding quantiles 160 # are insignificantly less than the value. 161 # 162 # For example, the value for infinite number of freedoms is 2.5758 163 # 164 # So, to reduce table size, we take this, greater value, 165 # overestimating inaccuracy for no more than 4%. 166 # 167 t_coef = t_gamma_n_m1 [50] 168 } 169 170 # 171 # Calculate inaccuracy estimation 172 # 173 sum_delta_squares = 0 174 for (i = 1; i <= n; i++) { 175 sum_delta_squares += (avg - a[i]) ^ 2 176 } 177 178 delta = t_coef * sqrt (sum_delta_squares / (n * (n - 1))) 179 180 print avg, delta 181 } else { 182 print avg 183 } 184 } 185 " || exit 1; 186 ); 187 calc_status=$? 188fi 189 190echo "$perf_values" 191 192if [ $? -ne 0 ]; 193then 194 exit 1; 195fi; 196