1#!/bin/sh 2 3#set -x 4 5cd $(dirname "$0") 6 7default_path=".." 8default_cmd="sparse \$file" 9default_args="$SPARSE_TEST_ARGS" 10tests_list="" 11prog_name=`basename $0` 12 13if [ ! -x "$default_path/sparse-llvm" ]; then 14 disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis" 15fi 16if [ ! -x "$default_path/scheck" ]; then 17 disabled_cmds="$disabled_cmds scheck" 18fi 19 20# flags: 21# - some tests gave an unexpected result 22failed=0 23 24# counts: 25# - tests that have not been converted to test-suite format 26# - tests that are disabled 27# - tests that passed 28# - tests that failed 29# - tests that failed but are known to fail 30unhandled_tests=0 31disabled_tests=0 32ok_tests=0 33ko_tests=0 34known_ko_tests=0 35 36# defaults to not verbose 37[ -z "$V" ] && V=0 38vquiet="" 39quiet=0 40abort=0 41 42 43## 44# verbose(string) - prints string if we are in verbose mode 45verbose() 46{ 47 [ "$V" -eq "1" ] && echo " $1" 48 return 0 49} 50 51## 52# warning(string) - prints a warning 53warning() 54{ 55 [ "$quiet" -ne 1 ] && echo "warning: $1" 56 return 0 57} 58 59## 60# error(string[, die]) - prints an error and exits with value die if given 61error() 62{ 63 [ "$quiet" -ne 1 ] && echo "error: $1" 64 [ -n "$2" ] && exit $2 65 return 0 66} 67 68 69## 70# get_tag_value(file) - get the 'check-<...>' tags & values 71get_tag_value() 72{ 73 check_name="" 74 check_command="$default_cmd" 75 check_exit_value=0 76 check_timeout=0 77 check_known_to_fail=0 78 check_error_ignore=0 79 check_output_ignore=0 80 check_output_contains=0 81 check_output_excludes=0 82 check_output_pattern=0 83 check_output_match=0 84 check_output_returns=0 85 check_arch_ignore="" 86 check_arch_only="" 87 check_assert="" 88 check_cpp_if="" 89 90 lines=$(grep '^ \* check-[a-z-]*' $1 | \ 91 sed -e 's/^ \* \(check-[a-z-]*:*\) *\(.*\)$/\1 \2/') 92 93 while read tag val; do 94 #echo "-> tag: '$tag'" 95 #echo "-> val: '$val'" 96 case $tag in 97 check-name:) check_name="$val" ;; 98 check-command:) check_command="$val" ;; 99 check-exit-value:) check_exit_value="$val" ;; 100 check-timeout:) [ -z "$val" ] && val=1 101 check_timeout="$val" ;; 102 check-known-to-fail) check_known_to_fail=1 ;; 103 check-error-ignore) check_error_ignore=1 ;; 104 check-output-ignore) check_output_ignore=1 ;; 105 check-output-contains:) check_output_contains=1 ;; 106 check-output-excludes:) check_output_excludes=1 ;; 107 check-output-pattern) check_output_pattern=1 ;; 108 check-output-match) check_output_match=1 ;; 109 check-output-returns:) check_output_returns=1 ;; 110 check-arch-ignore:) arch=$(uname -m) 111 check_arch_ignore="$val" ;; 112 check-arch-only:) arch=$(uname -m) 113 check_arch_only="$val" ;; 114 check-assert:) check_assert="$val" ;; 115 check-cpp-if:) check_cpp_if="$val" ;; 116 117 check-description:) ;; # ignore 118 check-note:) ;; # ignore 119 check-warning:) ;; # ignore 120 check-error-start) ;; # ignore 121 check-error-end) ;; # ignore 122 check-output-start) ;; # ignore 123 check-output-end) ;; # ignore 124 check-should-pass) ;; # ignore, unused annotation 125 check-should-fail) ;; # ignore, unused annotation 126 check-should-warn) ;; # ignore, unused annotation 127 check-*) error "$1: unknown tag '$tag'" 1 ;; 128 esac 129 done << EOT 130 $lines 131EOT 132} 133 134## 135# helper for has_(each|none)_patterns() 136has_patterns() 137{ 138 ifile="$1" 139 patt="$2" 140 ofile="$3" 141 cmp="$4" 142 msg="$5" 143 grep "$patt:" "$ifile" | \ 144 sed -e "s/^.*$patt: *\(.*\)$/\1/" | \ 145 while read val; do 146 grep -s -q "$val" "$ofile" 147 if [ "$?" $cmp 0 ]; then 148 error " Pattern '$val' unexpectedly $msg" 149 return 1 150 fi 151 done 152 153 return $? 154} 155 156## 157# has_each_patterns(ifile tag ofile) - does ofile contains some 158# of the patterns given by ifile's tags? 159# 160# returns 0 if all present, 1 otherwise 161has_each_patterns() 162{ 163 has_patterns "$1" "$2" "$4" -ne "$3" 164} 165 166## 167# has_none_patterns(ifile tag ofile) - does ofile contains some 168# of the patterns given by ifile's tags? 169# 170# returns 1 if any present, 0 otherwise 171has_none_patterns() 172{ 173 has_patterns "$1" "$2" "$4" -eq "$3" 174} 175 176## 177# minmax_patterns(ifile tag ofile) - does ofile contains the 178# the patterns given by ifile's tags 179# the right number of time? 180minmax_patterns() 181{ 182 ifile="$1" 183 patt="$2" 184 ofile="$3" 185 grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \ 186 sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \ 187 -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \ 188 while read min max pat; do 189 n=$(grep -s "$pat" "$ofile" | wc -l) 190 if [ "$max" = "eq" ]; then 191 if [ "$n" -ne "$min" ]; then 192 error " Pattern '$pat' expected $min times but got $n times" 193 return 1 194 fi 195 continue 196 fi 197 if [ "$min" != '-' ]; then 198 if [ "$n" -lt "$min" ]; then 199 error " Pattern '$pat' expected min $min times but got $n times" 200 return 1 201 fi 202 fi 203 if [ "$max" != '-' ]; then 204 if [ "$n" -gt "$max" ]; then 205 error " Pattern '$pat' expected max $max times but got $n times" 206 return 1 207 fi 208 fi 209 done 210 211 return $? 212} 213 214## 215match_patterns() 216{ 217 ifile="$1" 218 patt="$2" 219 ofile="$3" 220 grep "$patt" "$ifile" | sed -e "s/^.*$patt(\(.*\)): *\(.*\)$/\1 \2/" | \ 221 while read ins pat; do 222 grep -s "^ $ins\\.*[0-9]* " "$ofile" | grep -v -s -q "$pat" 223 if [ "$?" -ne 1 ]; then 224 error " IR doesn't match '$pat'" 225 return 1 226 fi 227 done 228 229 return $? 230} 231 232## 233return_patterns() 234{ 235 ifile="$1" 236 patt="$2" 237 ofile="$3" 238 grep "$patt:" "$ifile" | sed -e "s/^.*$patt: *\(.*\)$/\1/" | \ 239 while read ret; do 240 grep -s "^ ret\\.[0-9]" "$ofile" | grep -v -s -q "[ \$]${ret}\$" 241 if [ "$?" -ne 1 ]; then 242 error " Return doesn't match '$ret'" 243 return 1 244 fi 245 done 246 247 return $? 248} 249 250## 251# arg_file(filename) - checks if filename exists 252arg_file() 253{ 254 [ -z "$1" ] && { 255 do_usage 256 exit 1 257 } 258 [ -e "$1" ] || { 259 error "Can't open file $1" 260 exit 1 261 } 262 return 0 263} 264 265 266## 267do_usage() 268{ 269echo "$prog_name - a tiny automatic testing script" 270echo "Usage: $prog_name [option(s)] [command] [arguments]" 271echo 272echo "options:" 273echo " -a|--abort Abort the tests as soon as one fails." 274echo " -q|--quiet Be extra quiet while running the tests." 275echo " --args='...' Add these options to the test command." 276echo 277echo "commands:" 278echo " [file ...] Runs the test suite on the given file(s)." 279echo " If a directory is given, run only those files." 280echo " If no file is given, run the whole testsuite." 281echo " single file Run the test in 'file'." 282echo " format file [name [cmd]] Help writing a new test case using cmd." 283echo 284echo " [command] help Print usage." 285} 286 287disable() 288{ 289 disabled_tests=$(($disabled_tests + 1)) 290 if [ -z "$vquiet" ]; then 291 echo " SKIP $1 ($2)" 292 fi 293} 294 295## 296# do_test(file) - tries to validate a test case 297# 298# it "parses" file, looking for check-* tags and tries to validate 299# the test against an expected result 300# returns: 301# - 0 if the test passed, 302# - 1 if it failed, 303# - 2 if it is not a "test-suite" test. 304# - 3 if the test is disabled. 305do_test() 306{ 307 test_failed=0 308 file="$1" 309 quiet=0 310 311 get_tag_value $file 312 313 # can this test be handled by test-suite ? 314 # (it has to have a check-name key in it) 315 if [ "$check_name" = "" ]; then 316 warning "$file: test unhandled" 317 unhandled_tests=$(($unhandled_tests + 1)) 318 return 2 319 fi 320 test_name="$check_name" 321 322 # does the test provide a specific command ? 323 if [ "$check_command" = "" ]; then 324 check_command="$defaut_command" 325 fi 326 327 # check for disabled commands 328 set -- $check_command 329 base_cmd=$1 330 for i in $disabled_cmds; do 331 if [ "$i" = "$base_cmd" ] ; then 332 disable "$test_name" "$file" 333 return 3 334 fi 335 done 336 if [ "$check_arch_ignore" != "" ]; then 337 if echo $arch | egrep -q -w "$check_arch_ignore"; then 338 disable "$test_name" "$file" 339 return 3 340 fi 341 fi 342 if [ "$check_arch_only" != "" ]; then 343 if ! (echo $arch | egrep -q -w "$check_arch_only"); then 344 disable "$test_name" "$file" 345 return 3 346 fi 347 fi 348 if [ "$check_assert" != "" ]; then 349 res=$(../sparse - 2>&1 >/dev/null <<- EOF 350 _Static_assert($check_assert, "$check_assert"); 351 EOF 352 ) 353 if [ "$res" != "" ]; then 354 disable "$test_name" "$file" 355 return 3 356 fi 357 fi 358 if [ "$check_cpp_if" != "" ]; then 359 res=$(../sparse -E - 2>/dev/null <<- EOF 360 #if !($check_cpp_if) 361 fail 362 #endif 363 EOF 364 ) 365 if [ "$res" != "" ]; then 366 disable "$test_name" "$file" 367 return 3 368 fi 369 fi 370 371 if [ -z "$vquiet" ]; then 372 echo " TEST $test_name ($file)" 373 fi 374 375 verbose "Using command : $(echo "$@")" 376 377 # grab the expected exit value 378 expected_exit_value=$check_exit_value 379 verbose "Expecting exit value: $expected_exit_value" 380 381 # do we want a timeout? 382 pre_cmd="" 383 if [ $check_timeout -ne 0 ]; then 384 pre_cmd="timeout $check_timeout" 385 fi 386 387 shift 388 # launch the test command and 389 # grab the actual output & exit value 390 eval $pre_cmd $default_path/$base_cmd $default_args "$@" \ 391 1> $file.output.got 2> $file.error.got 392 actual_exit_value=$? 393 394 must_fail=$check_known_to_fail 395 [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1 396 known_ko_tests=$(($known_ko_tests + $must_fail)) 397 398 for stream in error output; do 399 eval ignore=\$check_${stream}_ignore 400 [ $ignore -eq 1 ] && continue 401 402 # grab the expected output 403 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \ 404 | grep -v check-$stream > "$file".$stream.expected 405 406 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff 407 if [ "$?" -ne "0" ]; then 408 error "actual $stream text does not match expected $stream text." 409 error "see $file.$stream.* for further investigation." 410 [ $quiet -ne 1 ] && cat "$file".$stream.diff 411 test_failed=1 412 fi 413 done 414 415 if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then 416 error "Actual exit value does not match the expected one." 417 error "expected $expected_exit_value, got $actual_exit_value." 418 test_failed=1 419 fi 420 421 # verify the 'check-output-contains/excludes' tags 422 if [ $check_output_contains -eq 1 ]; then 423 has_each_patterns "$file" 'check-output-contains' absent $file.output.got 424 if [ "$?" -ne "0" ]; then 425 test_failed=1 426 fi 427 fi 428 if [ $check_output_excludes -eq 1 ]; then 429 has_none_patterns "$file" 'check-output-excludes' present $file.output.got 430 if [ "$?" -ne "0" ]; then 431 test_failed=1 432 fi 433 fi 434 if [ $check_output_pattern -eq 1 ]; then 435 # verify the 'check-output-pattern(...)' tags 436 minmax_patterns "$file" 'check-output-pattern' $file.output.got 437 if [ "$?" -ne "0" ]; then 438 test_failed=1 439 fi 440 fi 441 if [ $check_output_match -eq 1 ]; then 442 # verify the 'check-output-match($insn): $patt' tags 443 match_patterns "$file" 'check-output-match' $file.output.got 444 if [ "$?" -ne "0" ]; then 445 test_failed=1 446 fi 447 fi 448 if [ $check_output_returns -eq 1 ]; then 449 # verify the 'check-output-return: $value' tags 450 return_patterns "$file" 'check-output-returns' $file.output.got 451 if [ "$?" -ne "0" ]; then 452 test_failed=1 453 fi 454 fi 455 456 if [ "$must_fail" -eq "1" ]; then 457 if [ "$test_failed" -eq "1" ]; then 458 [ -z "$vquiet" ] && \ 459 echo "info: XFAIL: test '$file' is known to fail" 460 else 461 echo "error: XPASS: test '$file' is known to fail but succeed!" 462 fi 463 else 464 if [ "$test_failed" -eq "1" ]; then 465 echo "error: FAIL: test '$file' failed" 466 else 467 [ "$V" -ne "0" ] && \ 468 echo "info: PASS: test '$file' passed" 469 fi 470 fi 471 472 if [ "$test_failed" -ne "$must_fail" ]; then 473 [ $abort -eq 1 ] && exit 1 474 test_failed=1 475 failed=1 476 fi 477 478 if [ "$test_failed" -eq "1" ]; then 479 ko_tests=$(($ko_tests + 1)) 480 else 481 ok_tests=$(($ok_tests + 1)) 482 rm -f $file.{error,output}.{expected,got,diff} 483 fi 484 return $test_failed 485} 486 487do_test_suite() 488{ 489 for i in $tests_list; do 490 do_test "$i" 491 done 492 493 OK=OK 494 [ $failed -eq 0 ] || OK=KO 495 496 # prints some numbers 497 tests_nr=$(($ok_tests + $ko_tests)) 498 echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed" 499 if [ "$known_ko_tests" -ne 0 ]; then 500 echo " $known_ko_tests of them are known to fail" 501 fi 502 if [ "$unhandled_tests" -ne "0" ]; then 503 echo " $unhandled_tests tests could not be handled by $prog_name" 504 fi 505 if [ "$disabled_tests" -ne "0" ]; then 506 echo " $disabled_tests tests were disabled" 507 fi 508} 509 510## 511do_format_help() { 512echo "Usage: $prog_name [option(s)] [--]format file [name [cmd]]" 513echo 514echo "options:" 515echo " -a append the created test to the input file" 516echo " -f write a test known to fail" 517echo " -l write a test for linearized code" 518echo " -r write a test for linearized code returning 1" 519echo " -p write a test for pre-processing" 520echo " -s write a test for symbolic checking" 521echo 522echo "argument(s):" 523echo " file file containing the test case(s)" 524echo " name name for the test case (defaults to file)" 525echo " cmd command to be used (defaults to 'sparse \$file')" 526} 527 528## 529# do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags 530do_format() 531{ 532 def_cmd="$default_cmd" 533 append=0 534 linear=0 535 fail=0 536 ret='' 537 538 while [ $# -gt 0 ] ; do 539 case "$1" in 540 -a) 541 append=1 ;; 542 -f) 543 fail=1 ;; 544 -l) 545 def_cmd='test-linearize -Wno-decl $file' 546 linear=1 ;; 547 -r) 548 def_cmd='test-linearize -Wno-decl $file' 549 ret=1 ;; 550 -p) 551 def_cmd='sparse -E $file' ;; 552 -s) 553 def_cmd='scheck $file' ;; 554 555 help|-*) 556 do_format_help 557 return 0 558 ;; 559 *) break ;; 560 esac 561 shift 562 continue 563 done 564 565 if [ $# -lt 1 -o $# -gt 3 ]; then 566 do_format_help 567 return 0 568 fi 569 570 arg_file "$1" || return 1 571 572 file="$1" 573 fname="$2" 574 [ -z "$fname" ] && fname="$(basename "$1" .c)" 575 fcmd="$3" 576 [ -z "$fcmd" ] && fcmd="$def_cmd" 577 578 cmd=`eval echo $default_path/$fcmd` 579 $cmd 1> $file.output.got 2> $file.error.got 580 fexit_value=$? 581 [ $append != 0 ] && exec >> $file 582 cat <<_EOF 583 584/* 585 * check-name: $fname 586_EOF 587 if [ "$fcmd" != "$default_cmd" ]; then 588 echo " * check-command: $fcmd" 589 fi 590 if [ "$fexit_value" -ne "0" ]; then 591 echo " * check-exit-value: $fexit_value" 592 fi 593 if [ $fail != 0 ]; then 594 echo " * check-known-to-fail" 595 fi 596 if [ "$ret" != '' ]; then 597 echo ' *' 598 echo ' * check-output-ignore' 599 echo " * check-output-returns: $ret" 600 rm -f "$file.output.got" 601 fi 602 if [ $linear != 0 ]; then 603 echo ' *' 604 echo ' * check-output-ignore' 605 echo ' * check-output-contains: xyz\\\\.' 606 echo ' * check-output-excludes: \\\\.' 607 fi 608 for stream in output error; do 609 if [ -s "$file.$stream.got" ]; then 610 echo " *" 611 echo " * check-$stream-start" 612 cat "$file.$stream.got" 613 echo " * check-$stream-end" 614 fi 615 done 616 echo " */" 617 return 0 618} 619 620## allow flags from environment 621set -- $SPARSE_TEST_FLAGS "$@" 622 623## process the flags 624while [ "$#" -gt "0" ]; do 625 case "$1" in 626 -a|--abort) 627 abort=1 628 ;; 629 -q|--quiet) 630 vquiet=1 631 ;; 632 --args=*) 633 default_args="${1#--args=}"; 634 ;; 635 636 single|--single) 637 arg_file "$2" 638 do_test "$2" 639 case "$?" in 640 0) echo "$2 passed !";; 641 1) echo "$2 failed !";; 642 2) echo "$2 can't be handled by $prog_name";; 643 esac 644 exit $failed 645 ;; 646 format|--format) 647 shift 648 do_format "$@" 649 exit 0 650 ;; 651 help) 652 do_usage 653 exit 1 654 ;; 655 656 *.c|*.cdoc) 657 tests_list="$tests_list $1" 658 ;; 659 *) 660 if [ ! -d "$1" ]; then 661 do_usage 662 exit 1 663 fi 664 tests_list="$tests_list $(find "$1" -name '*.c' | sort)" 665 ;; 666 esac 667 shift 668done 669 670if [ -z "$tests_list" ]; then 671 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort` 672fi 673 674do_test_suite 675exit $failed 676 677