1#!/bin/bash 2 3# Copyright JS Foundation and other contributors, http://js.foundation 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17trap "exit 2" INT 18 19function pr_err() { 20 echo -e "\e[91mError: $@\e[39m" 21} 22 23function exit_err() { 24 pr_err $@ 25 26 exit 1 27} 28 29# Check if the specified build supports memory statistics options 30function is_mem_stats_build() { 31 [ -x "$1" ] || fail_msg "Engine '$1' is not executable" 32 33 tmpfile=`mktemp` 34 "$1" --mem-stats $tmpfile 2>&1 | grep -- "Ignoring JERRY_INIT_MEM_STATS flag because of !JMEM_STATS configuration." 2>&1 > /dev/null 35 code=$? 36 rm $tmpfile 37 38 return $code 39} 40 41USAGE="Usage:\n tools/run-perf-test.sh OLD_ENGINE NEW_ENGINE REPEATS TIMEOUT BENCH_FOLDER [-m result-file-name.md]" 42 43if [ "$#" -lt 5 ] 44then 45 echo -e "${USAGE}" 46 exit_err "Argument number mismatch..." 47fi 48 49ENGINE_OLD="$1" 50ENGINE_NEW="$2" 51REPEATS="$3" 52TIMEOUT="$4" 53BENCH_FOLDER="$5" 54OUTPUT_FORMAT="$6" 55OUTPUT_FILE="$7" 56 57if [ "$#" -gt 5 ] 58then 59 if [ "${OUTPUT_FORMAT}" != "-m" ] 60 then 61 exit_err "Please, use '-m result-file-name.md' as last arguments" 62 fi 63 if [ -z "${OUTPUT_FILE}" ] 64 then 65 exit_err "Missing md file name. Please, define the filename. Ex.: '-m result-file-name.md'" 66 fi 67 68 rm -rf "${OUTPUT_FILE}" 69fi 70 71if [ "${REPEATS}" -lt 1 ] 72then 73 exit_err "REPEATS must be greater than 0" 74fi 75 76if [ "${TIMEOUT}" -lt 1 ] 77then 78 exit_err "TIMEOUT must be greater than 0" 79fi 80 81perf_n=0 82mem_n=0 83 84perf_rel_mult=1.0 85perf_rel_inaccuracy_tmp=0 86mem_rel_mult=1.0 87mem_rel_inaccuracy_tmp="-1" 88 89# Unicode "figure space" character 90FIGURE_SPACE=$(echo -e -n "\xE2\x80\x87") 91 92# Unicode "approximately equal" character 93APPROXIMATELY_EQUAL=$(echo -n -e "\xE2\x89\x88") 94 95function run-compare() 96{ 97 COMMAND=$1 98 PRE=$2 99 TEST=$3 100 PRECISION=$4 101 UNIT=$5 102 103 ABS_FP_FMT="%$((PRECISION + 4)).$((PRECISION))f$UNIT" 104 REL_FP_FMT="%0.3f" 105 REL_SHOW_PLUS_SIGN_FP_FMT="%+0.3f" 106 107 OLD=$(timeout "${TIMEOUT}" ${COMMAND} "${ENGINE_OLD}" "${TEST}") || return 1 108 NEW=$(timeout "${TIMEOUT}" ${COMMAND} "${ENGINE_NEW}" "${TEST}") || return 1 109 110 #check result 111 ! $OLD || ! $NEW || return 1 112 113 OLD_value=$(echo "$OLD " | cut -d ' ' -f 1) 114 OLD_inaccuracy=$(echo "$OLD " | cut -d ' ' -f 2) 115 116 NEW_value=$(echo "$NEW " | cut -d ' ' -f 1) 117 NEW_inaccuracy=$(echo "$NEW " | cut -d ' ' -f 2) 118 119 #calc relative speedup 120 eval "rel_mult=\$${PRE}_rel_mult" 121 122 rel=$(echo "${OLD_value}" "${NEW_value}" | awk '{ print $2 / $1; }') 123 124 #increment n 125 ((${PRE}_n++)) 126 127 #calc percent to display 128 PERCENT=$(echo "$rel" | awk '{print (1.0 - $1) * 100; }') 129 130 if [[ "$OLD_inaccuracy" != "" && "$NEW_inaccuracy" != "" ]] 131 then 132 DIFF=$(printf "$ABS_FP_FMT -> $ABS_FP_FMT" $OLD_value $NEW_value) 133 rel_inaccuracy=$(echo "$OLD_value $OLD_inaccuracy $NEW_value $NEW_inaccuracy" | \ 134 awk "{ 135 OLD_value=\$1 136 OLD_inaccuracy=\$2 137 NEW_value=\$3 138 NEW_inaccuracy=\$4 139 140 rel_inaccuracy = (NEW_value / OLD_value) * sqrt ((OLD_inaccuracy / OLD_value) ^ 2 + (NEW_inaccuracy / NEW_value) ^ 2) 141 if (rel_inaccuracy < 0) { 142 rel_inaccuracy = -rel_inaccuracy 143 } 144 145 print rel_inaccuracy 146 }") 147 PERCENT_inaccuracy=$(echo "$rel_inaccuracy" | awk '{ print $1 * 100.0 }') 148 149 ext=$(echo "$PERCENT $PERCENT_inaccuracy" | \ 150 awk "{ 151 PERCENT=\$1 152 PERCENT_inaccuracy=\$2 153 154 if (PERCENT > 0.0 && PERCENT > PERCENT_inaccuracy) { 155 print \"[+]\" 156 } else if (PERCENT < 0 && -PERCENT > PERCENT_inaccuracy) { 157 print \"[-]\" 158 } else { 159 print \"[$APPROXIMATELY_EQUAL]\" 160 } 161 }") 162 163 if [[ $rel_inaccuracy_tmp -lt 0 ]] 164 then 165 return 1 166 fi 167 168 eval "rel_inaccuracy_tmp=\$${PRE}_rel_inaccuracy_tmp" 169 170 rel_inaccuracy_tmp=$(echo "$rel $rel_inaccuracy $rel_inaccuracy_tmp" | \ 171 awk "{ 172 rel=\$1 173 rel_inaccuracy=\$2 174 rel_inaccuracy_tmp=\$3 175 print rel_inaccuracy_tmp + (rel_inaccuracy / rel) ^ 2 176 }") 177 178 eval "${PRE}_rel_inaccuracy_tmp=\$rel_inaccuracy_tmp" 179 180 PERCENT=$(printf "%8s %11s" $(printf "$REL_SHOW_PLUS_SIGN_FP_FMT%%" $PERCENT) $(printf "(+-$REL_FP_FMT%%)" $PERCENT_inaccuracy)) 181 PERCENT="$PERCENT : $ext" 182 183 if [ "${OUTPUT_FORMAT}" == "-m" ] 184 then 185 WIDTH=42 186 MD_DIFF=$(printf "%s%s" "$DIFF" "$(printf "%$(($WIDTH - ${#DIFF}))s")") 187 MD_PERCENT=$(printf "%s%s" "$(printf "%$(($WIDTH - ${#PERCENT}))s")" "$PERCENT") 188 189 MD_FORMAT="\`%s\`<br>\`%s\`" 190 fi 191 192 CONSOLE_FORMAT="%20s : %19s" 193 else 194 ext="" 195 196 if [[ "$OLD_inaccuracy" != "" || "$NEW_inaccuracy" != "" ]] 197 then 198 return 1; 199 fi 200 201 DIFF=$(printf "$ABS_FP_FMT -> $ABS_FP_FMT" $OLD_value $NEW_value) 202 PERCENT=$(printf "$REL_SHOW_PLUS_SIGN_FP_FMT%%" $PERCENT) 203 204 if [ "${OUTPUT_FORMAT}" == "-m" ] 205 then 206 WIDTH=20 207 MD_DIFF=$(printf "%s%s" "$DIFF" "$(printf "%$(($WIDTH - ${#DIFF}))s")") 208 MD_PERCENT=$(printf "%s%s" "$(printf "%$(($WIDTH - ${#PERCENT}))s")" "$PERCENT") 209 210 MD_FORMAT="\`%s\`<br>\`%s\`" 211 fi 212 213 CONSOLE_FORMAT="%14s : %8s" 214 fi 215 216 rel_mult=$(echo "$rel_mult" "$rel" | awk '{print $1 * $2;}') 217 218 eval "${PRE}_rel_mult=\$rel_mult" 219 220 if [ "${OUTPUT_FORMAT}" == "-m" ] 221 then 222 printf "$MD_FORMAT" "$MD_DIFF" "$MD_PERCENT" | sed "s/ /$FIGURE_SPACE/g" >> "${OUTPUT_FILE}" 223 fi 224 225 printf "$CONSOLE_FORMAT" "$DIFF" "$PERCENT" 226} 227 228function run-test() 229{ 230 TEST=$1 231 232 # print only filename 233 if [ "${OUTPUT_FORMAT}" == "-m" ] 234 then 235 printf "%s | " "${TEST##*/}" >> "${OUTPUT_FILE}" 236 fi 237 238 printf "%50s | " "${TEST##*/}" 239 240 if [ "$IS_MEM_STAT" -ne 0 ] 241 then 242 run-compare "./tools/mem-stats-measure.sh" "mem" "${TEST}" 0 || return 1 243 else 244 run-compare "./tools/rss-measure.sh" "mem" "${TEST}" 0 k || return 1 245 fi 246 247 if [ "${OUTPUT_FORMAT}" == "-m" ] 248 then 249 printf " | " >> "${OUTPUT_FILE}" 250 fi 251 252 printf " | " 253 run-compare "./tools/perf.sh ${REPEATS}" "perf" "${TEST}" 3 s || return 1 254 255 if [ "${OUTPUT_FORMAT}" == "-m" ] 256 then 257 printf "\n" >> "${OUTPUT_FILE}" 258 fi 259 260 printf "\n" 261} 262 263function run-suite() 264{ 265 FOLDER=$1 266 267 for BENCHMARK in ${FOLDER}/*.js 268 do 269 run-test "${BENCHMARK}" 2> /dev/null || printf "<FAILED>\n" "${BENCHMARK}"; 270 done 271} 272 273date 274 275is_mem_stats_build "${ENGINE_OLD}" || is_mem_stats_build "${ENGINE_NEW}" 276IS_MEM_STAT=$? 277 278if [ "${OUTPUT_FORMAT}" == "-m" ] 279then 280 if [ "$IS_MEM_STAT" -ne 0 ] 281 then 282 echo "Benchmark | Peak alloc.<br>(+ is better) | Perf<br>(+ is better)" >> "${OUTPUT_FILE}" 283 else 284 echo "Benchmark | RSS<br>(+ is better) | Perf<br>(+ is better)" >> "${OUTPUT_FILE}" 285 fi 286 echo "---------: | --------- | ---------" >> "${OUTPUT_FILE}" 287fi 288 289if [ "$IS_MEM_STAT" -ne 0 ] 290then 291 printf "%50s | %25s | %35s\n" "Benchmark" "Peak alloc.(+ is better)" "Perf(+ is better)" 292else 293 printf "%50s | %25s | %35s\n" "Benchmark" "RSS(+ is better)" "Perf(+ is better)" 294fi 295 296run-suite "${BENCH_FOLDER}" 297 298mem_rel_gmean=$(echo "$mem_rel_mult" "$mem_n" | awk '{print $1 ^ (1.0 / $2);}') 299mem_percent_gmean=$(echo "$mem_rel_gmean" | awk '{print (1.0 - $1) * 100;}') 300if [[ $mem_rel_inaccuracy_tmp != "-1" ]] 301then 302 exit_err "Incorrect inaccuracy calculation for memory consumption geometric mean" 303fi 304 305perf_rel_gmean=$(echo "$perf_rel_mult" "$perf_n" | awk '{print $1 ^ (1.0 / $2);}') 306perf_percent_gmean=$(echo "$perf_rel_gmean" | awk '{print (1.0 - $1) * 100;}') 307if [[ "$perf_rel_inaccuracy_tmp" == "-1" ]] 308then 309 exit_err "Incorrect inaccuracy calculation for performance geometric mean" 310else 311 perf_percent_inaccuracy=$(echo "$perf_rel_gmean $perf_rel_inaccuracy_tmp $perf_n" | \ 312 awk "{ 313 perf_rel_gmean=\$1 314 perf_rel_inaccuracy_tmp=\$2 315 perf_n=\$3 316 317 print 100.0 * (perf_rel_gmean ^ (1.0 / perf_n) * sqrt (perf_rel_inaccuracy_tmp) / perf_n) 318 }") 319 perf_ext=$(echo "$perf_percent_gmean $perf_percent_inaccuracy" | \ 320 awk "{ 321 perf_percent_gmean=\$1 322 perf_percent_inaccuracy=\$2 323 324 if (perf_percent_gmean > 0.0 && perf_percent_gmean > perf_percent_inaccuracy) { 325 print \"[+]\" 326 } else if (perf_percent_gmean < 0 && -perf_percent_gmean > perf_percent_inaccuracy) { 327 print \"[-]\" 328 } else { 329 print \"[$APPROXIMATELY_EQUAL]\" 330 } 331 }") 332 perf_percent_inaccuracy=$(printf "(+-%0.3f%%) : $perf_ext" $perf_percent_inaccuracy) 333fi 334 335gmean_label_text="Geometric mean:" 336 337if [ "${OUTPUT_FORMAT}" == "-m" ] 338then 339 mem_percent_gmean_text=$(printf "RSS reduction: \`%0.3f%%\`" "$mem_percent_gmean") 340 perf_percent_gmean_text=$(printf "Speed up: \`%0.3f%% %s\`" "$perf_percent_gmean" "$perf_percent_inaccuracy") 341 printf "%s | %s | %s\n" "$gmean_label_text" "$mem_percent_gmean_text" "$perf_percent_gmean_text" >> "${OUTPUT_FILE}" 342fi 343 344mem_percent_gmean_text=$(printf "RSS reduction: %0.3f%%" "$mem_percent_gmean") 345perf_percent_gmean_text=$(printf "Speed up: %0.3f%% %s" "$perf_percent_gmean" "$perf_percent_inaccuracy") 346printf "%50s | %25s | %51s\n" "$gmean_label_text" "$mem_percent_gmean_text" "$perf_percent_gmean_text" 347 348date 349