1#!/bin/bash 2# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15set -e 16 17for ARGUMENT in "$@" 18do 19case "$ARGUMENT" in 20 --binary-dir=*) 21 PANDA_BINARY_ROOT="${ARGUMENT#*=}" 22 ;; 23 --root-dir=*) 24 PANDA_ROOT="${ARGUMENT#*=}" 25 ;; 26 --target-arch=*) 27 TARGET_ARCH="${ARGUMENT#*=}" 28 ;; 29 --paoc-mode=*) 30 PAOC_MODE="${ARGUMENT#*=}" 31 ;; 32 -build-panda) 33 BUILD_PANDA=true 34 ;; 35esac 36done 37 38if [ "$BUILD_PANDA" = true ]; then 39 cd $PANDA_BINARY_ROOT 40 make -j 41 cd - 42fi 43 44TARGET_NAME=NULL 45if [ "$TARGET_ARCH" = arm64 ]; then 46 TARGET_NAME=ARM64 47elif [ "$TARGET_ARCH" = arm ]; then 48 TARGET_NAME=ARM32 49elif [ "$TARGET_ARCH" = x86_64 ]; then 50 TARGET_NAME=X86_64 51else 52 echo "Unsupported arch: $TARGET_ARCH" 53 exit 1 54fi 55 56PAOC_MODE_NAME=NULL 57if [ "$PAOC_MODE" = aot ]; then 58 PAOC_MODE_NAME=AOT 59elif [ "$PAOC_MODE" = jit ]; then 60 PAOC_MODE_NAME=JIT 61elif [ "$PAOC_MODE" = osr ]; then 62 PAOC_MODE_NAME=OSR 63else 64 echo "Unsupported paoc-mode: $PAOC_MODE" 65 exit 1 66fi 67 68TEMP_DIR=$(pwd)/benchmark_coverage_${PAOC_MODE_NAME}_${TARGET_NAME} 69mkdir -p $TEMP_DIR 70cd $TEMP_DIR 71 72PANDASM=$PANDA_BINARY_ROOT/bin/ark_asm 73PAOC=$PANDA_BINARY_ROOT/bin/ark_aot 74BENCHMARKS=$PANDA_ROOT/tests/benchmarks 75SMALL_TESTS=$PANDA_ROOT/tests/cts-assembly 76 77ARKSTDLIB="$PANDA_BINARY_ROOT/pandastdlib/arkstdlib.abc" 78 79MAX_BYTECODE_SIZE=40000 80 81IGNORE_FUNCTIONS=$PANDA_ROOT/compiler/tools/ignore_checked_coverage.txt 82LOG_ERROR=$PANDA_BINARY_ROOT/compiler/coverage/log_compiler_error_${PAOC_MODE}_${TARGET_ARCH}.txt 83HTML=$PANDA_BINARY_ROOT/compiler/coverage/BenchmarkCoverage_${PAOC_MODE}_${TARGET_ARCH}.html 84mkdir -p $PANDA_BINARY_ROOT/compiler/coverage 85 86[ -e "$HTML" ] && rm $HTML 87[ -e "$LOG_ERROR" ] && rm $LOG_ERROR 88 89critical_compiler_failures=() 90not_compiled_benchmarks=() 91 92# Checks if a test is in the ignore list: 93function check_critical_failure() { 94 status=0 95 local filtered_ignore_list=$(grep "^IGNORE.*$1" $IGNORE_FUNCTIONS) || status=$? 96 # Check if the ignore list contains lines with current benchmark/CTS-test name: 97 if [ "${status}" -ne 0 ]; then 98 critical_compiler_failures+=( "$1:$name" ) 99 return 100 fi 101 while IFS= read -r line; do 102 # Check if methods were specified: 103 if (! echo "$line" | grep -q "FUNCS") || (echo "$line" | grep -q "$name"); then 104 # Check architecture ignore compound: 105 if (! echo "$line" | grep -q "ARM64\|ARM32\|X86_64") || (echo "$line" | grep -q "$TARGET_NAME"); then 106 # Check compilation mode compound: 107 if (! echo "$line" | grep -q "OSR\|JIT") || (echo "$line" | grep -q "$PAOC_MODE_NAME"); then 108 return 109 fi 110 fi 111 fi 112 done <<< "$filtered_ignore_list" 113 critical_compiler_failures+=( "$1:$name" ) 114 return 115} 116 117function separate_method_name() { 118 if [[ "$1" == *\.* ]]; then 119 method_name=$(echo $1 | rev | cut -f 1 -d '.' | rev) 120 class_name=$(echo $1 | rev | cut -f 2- -d '.' | rev) 121 else 122 class_name="_GLOBAL" 123 method_name=$1 124 fi 125} 126 127function calculate_benchmark_coverage() { 128 local all_functions=0 129 local all_compiled=0 130 local all_assert=0 131 132 echo "<header><h1>Benchmark coverage statistic</h1></header>" >> $HTML 133 134 for benchmark in $(find $BENCHMARKS/*\.pa -maxdepth 0 -exec basename -s .pa {} \;) 135 do 136 # Сhecking that benchmark has compiled 137 if $PANDASM $BENCHMARKS/$benchmark.pa $benchmark.abc 138 then 139 local method_names=`grep "^[.]function" $BENCHMARKS/$benchmark.pa \ 140 | grep -v "<[^>]*external[^>]*>" | cut -f3 -d' ' | cut -f1 -d'('` 141 local functions=0 142 local compiled=0 143 local assert=0 144 local func_status=() 145 146 for name in $method_names 147 do 148 let "functions+=1" 149 # Get class and method names 150 separate_method_name $name 151 local tmp_file=$benchmark-$class_name-$method_name.tmp 152 rm -f $tmp_file 153 local compiler_status=0 154 $PAOC --paoc-panda-files="$benchmark.abc" --compiler-regex="$class_name::$method_name" \ 155 --compiler-ignore-failures=false --paoc-mode=$PAOC_MODE \ 156 --boot-panda-files="$ARKSTDLIB" --load-runtimes="core" \ 157 --compiler-max-bytecode-size=$MAX_BYTECODE_SIZE \ 158 --compiler-cross-arch=$TARGET_ARCH --log-level=debug --log-components=compiler > $tmp_file 2>&1 || \ 159 compiler_status=$? 160 if [ "$compiler_status" -eq 0 ] 161 then 162 let "compiled+=1" 163 func_status+=( "compiled" ) 164 else 165 if grep -q "RunOptimizations failed!" "$tmp_file" 166 then 167 func_status+=( "optimizations" ) 168 169 elif grep -q "IrBuilder failed!" "$tmp_file" 170 then 171 func_status+=( "ir_builder" ) 172 else 173 let "assert+=1" 174 func_status+=( "assert" ) 175 fi 176 # Checking that the benchmark function should be built and not have asserts 177 check_critical_failure $benchmark 178 # Information for reproducing the error 179 echo "Benchmark=$benchmark" >> $LOG_ERROR 180 echo "Function=$name" >> $LOG_ERROR 181 echo "Status=${func_status[-1]}" >> $LOG_ERROR 182 echo "Reproduce:" >> $LOG_ERROR 183 echo "$PANDASM $BENCHMARKS/$benchmark.pa $benchmark.abc" >> $LOG_ERROR 184 echo -n "$PAOC --paoc-panda-files=\"$benchmark.abc\" --paoc-mode=$PAOC_MODE" >> $LOG_ERROR 185 echo -n " --compiler-regex=\"$class_name::$method_name\"" >> $LOG_ERROR 186 echo -n " --boot-panda-files=\"$ARKSTDLIB\" --load-runtimes=\"core\"" >> $LOG_ERROR 187 echo -n " --compiler-ignore-failures=false" >> $LOG_ERROR 188 echo -n " --compiler-cross-arch=$TARGET_ARCH" >> $LOG_ERROR 189 echo -n " --compiler-max-bytecode-size=$MAX_BYTECODE_SIZE" >> $LOG_ERROR 190 echo " --log-level=debug --log-components=compiler" >> $LOG_ERROR 191 echo "" >> $LOG_ERROR 192 fi 193 rm -f $tmp_file 194 done 195 rm -f $benchmark.abc 196 197 # Benchmark status 198 if [ $assert -ne 0 ] || [ $compiled -eq 0 ] 199 then 200 local background="#FF250D" 201 elif [ $compiled -ne $functions ] 202 then 203 local background="yellow" 204 else 205 local background="lime" 206 fi 207 # Benchmark statistics 208 local coverage=$(($compiled * 100 / $functions)) 209 echo "<p><b><span style=\"background-color:$background\">$benchmark</span></b> 210 functions = $functions, 211 compiled = $compiled, 212 assert = $assert, 213 coverage = $coverage%</p>" >> $HTML 214 215 # Names benchmark functions 216 echo "<table cellpadding=\"5\"><tr> 217 <th align=\"center\">functions</th>" >> $HTML 218 for name in $method_names 219 do 220 echo "<td align=\"center\">$name</td>" >> $HTML 221 done 222 223 # Statuses of benchmark functions 224 echo "<tr><th align=\"center\">status</th>" >> $HTML 225 for status in "${func_status[@]}" 226 do 227 echo "<td align=\"center\" bgcolor=" >> $HTML 228 if [ "$status" == "compiled" ] 229 then 230 echo "\"#d7e7a9\">compiled" >> $HTML 231 elif [ "$status" == "ir_builder" ] 232 then 233 echo "\"salmon\">ir_builder" >> $HTML 234 elif [ "$status" == "optimizations" ] 235 then 236 echo "\"salmon\">optimizations" >> $HTML 237 else 238 echo "\"#F5001D\">assert" >> $HTML 239 fi 240 done 241 echo "</table>" >> $HTML 242 243 all_functions=$((all_functions+$functions)) 244 all_compiled=$((all_compiled+$compiled)) 245 all_assert=$((all_assert+$assert)) 246 else 247 not_compiled_benchmarks+=( "$benchmark" ) 248 fi 249 done 250 251 # Benchmark report 252 local benchmark_coverage=$(($all_compiled * 100 / $all_functions)) 253 echo "Benchmark report $TARGET_ARCH:" 254 echo "Functions = $all_functions | Compiled = $all_compiled | Assert = $all_assert | Coverage = $benchmark_coverage%" 255} 256 257function calculate_assembly_tests_coverage() { 258 local all_tests=0 259 local success_tests=0 260 261 # CTS-assembly test 262 echo "<header><h1>CTS-ASSEMBLY TESTS</h1></header> 263 <table cellpadding=\"5\">" >> $HTML 264 265 for test in $(ls $SMALL_TESTS | cut -f1 -d'.') 266 do 267 # Сhecking that tests has compiled 268 if $PANDASM $SMALL_TESTS/$test.pa $test.abc 269 then 270 local method_names=$(grep "^[.]function" $SMALL_TESTS/$test.pa | grep -v "<[^>]*external[^>]*>\|<[^>]*noimpl[^>]*>" | cut -f3 -d' ' | cut -f1 -d'(') 271 local functions=0 272 local compiled=0 273 local assert=0 274 275 for name in $method_names 276 do 277 let "functions+=1" 278 279 local is_cctor=1 280 local is_ctor=1 281 grep -q "${name}()\s*<.*cctor.*>" $SMALL_TESTS/$test.pa || local is_cctor=$(($?^1)) 282 grep -q "${name}(.*)\s*<.*ctor.*>" $SMALL_TESTS/$test.pa || local is_ctor=$(($?^1)) 283 # Get class and method names 284 separate_method_name $name 285 local method_name=$method_name 286 if [ "$is_cctor" -eq 1 ] 287 then 288 local method_name=".cctor" 289 elif [ "$is_ctor" -eq 1 ] 290 then 291 local method_name=".ctor" 292 fi 293 local load_runtimes="core" 294 local boot_panda_files=$ARKSTDLIB 295 296 local tmp_file=$test-$class_name-$method_name.tmp 297 rm -f $tmp_file 298 local compiler_status=0 299 $PAOC --paoc-panda-files=$test.abc --compiler-regex="$class_name::$method_name" \ 300 --compiler-ignore-failures=false --paoc-mode=$PAOC_MODE \ 301 --boot-panda-files="$boot_panda_files" \ 302 --load-runtimes="$load_runtimes" \ 303 --compiler-max-bytecode-size=$MAX_BYTECODE_SIZE \ 304 --compiler-cross-arch=$TARGET_ARCH --log-level=debug --log-components=compiler > $tmp_file 2>&1 || \ 305 compiler_status=$? 306 if [ "$compiler_status" -eq 0 ] 307 then 308 let "compiled+=1" 309 else 310 if grep -q "RunOptimizations failed!" "$tmp_file" 311 then 312 local test_status="optimizations" 313 elif grep -q "IrBuilder failed!" "$tmp_file" 314 then 315 local test_status="ir_builder" 316 else 317 let "assert+=1" 318 local test_status="assert" 319 fi 320 # Checking that the benchmark function should be built and not have asserts 321 check_critical_failure $test 322 # Information for reproducing the error 323 echo "CTS-assembly test=$test" >> $LOG_ERROR 324 echo "Function=$name" >> $LOG_ERROR 325 echo "Status=$test_status" >> $LOG_ERROR 326 echo "Reproduce:" >> $LOG_ERROR 327 echo "$PANDASM $SMALL_TESTS/$test.pa $test.abc" >> $LOG_ERROR 328 echo -n "$PAOC --paoc-panda-files=\"$test.abc\" --paoc-mode=$PAOC_MODE" >> $LOG_ERROR 329 echo -n " --compiler-regex=\"$class_name::$method_name\"" >> $LOG_ERROR 330 echo -n " --compiler-ignore-failures=false" >> $LOG_ERROR 331 echo -n " --boot-panda-files=\"$boot_panda_files\"" >> $LOG_ERROR 332 echo -n " --load-runtimes=\"$load_runtimes\"" >> $LOG_ERROR 333 echo -n " --compiler-cross-arch=$TARGET_ARCH" >> $LOG_ERROR 334 echo -n " --compiler-max-bytecode-size=$MAX_BYTECODE_SIZE" >> $LOG_ERROR 335 echo " --log-level=debug --log-components=compiler --compiler-log=all" >> $LOG_ERROR 336 echo "" >> $LOG_ERROR 337 fi 338 rm -f $tmp_file 339 done 340 rm -f $test.abc 341 342 local indent=$(($all_tests % 10)) 343 if [ $indent -eq 0 ] 344 then 345 echo "<tr>" >> $HTML 346 fi 347 348 # Test status 349 if [ $assert -ne 0 ] || [ $compiled -eq 0 ] 350 then 351 local background="red" 352 elif [ $compiled -ne $functions ] 353 then 354 local background="yellow" 355 else 356 local background="#d7e7a9" 357 let "success_tests+=1" 358 fi 359 echo "<td align=\"center\" bgcolor=$background>$test</td>" >> $HTML 360 361 let "all_tests+=1" 362 fi 363 done 364 echo "</table>" >> $HTML 365 366 # CTS-assembly tests report 367 local coverage=$(($success_tests * 100 / $all_tests)) 368 echo "<h4>Coverage tests = $coverage%</h4>" >> $HTML 369 echo "CTS-assembly tests coverage = $coverage%" 370} 371 372 373echo "<!DOCTYPE html> 374<html> 375 <head> 376 <style>table, th, td {border: 1px solid black; border-collapse: collapse;}</style> 377 <title>Benchmark coverage statistic</title> 378 </head> 379 <body>" >> $HTML 380 381calculate_benchmark_coverage 382 383calculate_assembly_tests_coverage 384 385rm -r "$TEMP_DIR" 386 387if [ ${#critical_compiler_failures[@]} -ne 0 ] 388then 389 echo "The following functions must be compiled:" 390 for failure in ${critical_compiler_failures[@]} 391 do 392 echo " $failure" 393 done 394 echo "If you are sure that your actions are correct: 395 * add the ignore flag for the specified functions in the $IGNORE_FUNCTIONS 396 * create an issue the error description with assignee @pishin" 397 echo "To reproduce the error, see: $LOG_ERROR" 398 exit 1 399fi 400 401echo "<h3>Assembler failed to build the following benchmarks:</h3>" >> $HTML 402for benchmark in "${not_compiled_benchmarks[@]}" 403do 404 echo "<p>$benchmark</p>" >> $HTML 405done 406 407 408echo "<h3>Benchmark status</h3> 409 <table cellpadding=\"5\"> 410 <tr> 411 <th align=\"left\">Compiled all function</th> 412 <td bgcolor=\"lime\"></td> 413 </tr> 414 <tr> 415 <th align=\"left\">Compiled some function and doesn't have assert</th> 416 <td bgcolor=\"yellow\"></td> 417 </tr> 418 <tr> 419 <th align=\"left\">Has assert or all functions are not compiled</th> 420 <td bgcolor=\"#FF250D\"></td> 421 </tr> 422 </table> 423 <h3>Function status</h3> 424 <table cellpadding=\"5\"> 425 <tr> 426 <th align=\"left\">Successfully compiled</th> 427 <td align=\"center\" bgcolor=\"#d7e7a9\">compiled</td> 428 </tr> 429 <tr> 430 <th align=\"left\">Failed to build ir</th> 431 <td align=\"center\" bgcolor=\"salmon\">ir_builder</td> 432 </tr> 433 <tr> 434 <th align=\"left\">Optimization failed</th> 435 <td align=\"center\" bgcolor=\"salmon\">optimizations</td> 436 </tr> 437 <tr> 438 <th align=\"left\">ASSERT thrown</th> 439 <td align=\"center\" bgcolor=\"#F5001D\">assert</td> 440 </tr> 441 </table> 442 </body> 443</html>" >> $HTML 444