1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Run a series of tests under KVM. By default, this series is specified 5# by the relevant CFLIST file, but can be overridden by the --configs 6# command-line argument. 7# 8# Usage: kvm.sh [ options ] 9# 10# Copyright (C) IBM Corporation, 2011 11# 12# Authors: Paul E. McKenney <paulmck@linux.ibm.com> 13 14scriptname=$0 15args="$*" 16 17T=${TMPDIR-/tmp}/kvm.sh.$$ 18trap 'rm -rf $T' 0 19mkdir $T 20 21cd `dirname $scriptname`/../../../../../ 22 23dur=$((30*60)) 24dryrun="" 25KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM 26PATH=${KVM}/bin:$PATH; export PATH 27. functions.sh 28 29TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`" 30TORTURE_DEFCONFIG=defconfig 31TORTURE_BOOT_IMAGE="" 32TORTURE_INITRD="$KVM/initrd"; export TORTURE_INITRD 33TORTURE_KCONFIG_ARG="" 34TORTURE_KCONFIG_GDB_ARG="" 35TORTURE_BOOT_GDB_ARG="" 36TORTURE_QEMU_GDB_ARG="" 37TORTURE_KCONFIG_KASAN_ARG="" 38TORTURE_KCONFIG_KCSAN_ARG="" 39TORTURE_KMAKE_ARG="" 40TORTURE_QEMU_MEM=512 41TORTURE_SHUTDOWN_GRACE=180 42TORTURE_SUITE=rcu 43TORTURE_TRUST_MAKE="" 44resdir="" 45configs="" 46cpus=0 47ds=`date +%Y.%m.%d-%H.%M.%S` 48jitter="-1" 49 50usage () { 51 echo "Usage: $scriptname optional arguments:" 52 echo " --allcpus" 53 echo " --bootargs kernel-boot-arguments" 54 echo " --bootimage relative-path-to-kernel-boot-image" 55 echo " --buildonly" 56 echo " --configs \"config-file list w/ repeat factor (3*TINY01)\"" 57 echo " --cpus N" 58 echo " --datestamp string" 59 echo " --defconfig string" 60 echo " --dryrun sched|script" 61 echo " --duration minutes" 62 echo " --gdb" 63 echo " --help" 64 echo " --interactive" 65 echo " --jitter N [ maxsleep (us) [ maxspin (us) ] ]" 66 echo " --kconfig Kconfig-options" 67 echo " --kmake-arg kernel-make-arguments" 68 echo " --mac nn:nn:nn:nn:nn:nn" 69 echo " --memory megabytes|nnnG" 70 echo " --no-initrd" 71 echo " --qemu-args qemu-arguments" 72 echo " --qemu-cmd qemu-system-..." 73 echo " --results absolute-pathname" 74 echo " --torture lock|rcu|rcuscale|refscale|scf" 75 echo " --trust-make" 76 exit 1 77} 78 79while test $# -gt 0 80do 81 case "$1" in 82 --allcpus) 83 cpus=$TORTURE_ALLOTED_CPUS 84 max_cpus=$TORTURE_ALLOTED_CPUS 85 ;; 86 --bootargs|--bootarg) 87 checkarg --bootargs "(list of kernel boot arguments)" "$#" "$2" '.*' '^--' 88 TORTURE_BOOTARGS="$2" 89 shift 90 ;; 91 --bootimage) 92 checkarg --bootimage "(relative path to kernel boot image)" "$#" "$2" '[a-zA-Z0-9][a-zA-Z0-9_]*' '^--' 93 TORTURE_BOOT_IMAGE="$2" 94 shift 95 ;; 96 --buildonly) 97 TORTURE_BUILDONLY=1 98 ;; 99 --configs|--config) 100 checkarg --configs "(list of config files)" "$#" "$2" '^[^/]*$' '^--' 101 configs="$2" 102 shift 103 ;; 104 --cpus) 105 checkarg --cpus "(number)" "$#" "$2" '^[0-9]*$' '^--' 106 cpus=$2 107 TORTURE_ALLOTED_CPUS="$2" 108 max_cpus="`identify_qemu_vcpus`" 109 if test "$TORTURE_ALLOTED_CPUS" -gt "$max_cpus" 110 then 111 TORTURE_ALLOTED_CPUS=$max_cpus 112 fi 113 shift 114 ;; 115 --datestamp) 116 checkarg --datestamp "(relative pathname)" "$#" "$2" '^[^/]*$' '^--' 117 ds=$2 118 shift 119 ;; 120 --defconfig) 121 checkarg --defconfig "defconfigtype" "$#" "$2" '^[^/][^/]*$' '^--' 122 TORTURE_DEFCONFIG=$2 123 shift 124 ;; 125 --dryrun) 126 checkarg --dryrun "sched|script" $# "$2" 'sched\|script' '^--' 127 dryrun=$2 128 shift 129 ;; 130 --duration) 131 checkarg --duration "(minutes)" $# "$2" '^[0-9]*$' '^error' 132 dur=$(($2*60)) 133 shift 134 ;; 135 --gdb) 136 TORTURE_KCONFIG_GDB_ARG="CONFIG_DEBUG_INFO=y"; export TORTURE_KCONFIG_GDB_ARG 137 TORTURE_BOOT_GDB_ARG="nokaslr"; export TORTURE_BOOT_GDB_ARG 138 TORTURE_QEMU_GDB_ARG="-s -S"; export TORTURE_QEMU_GDB_ARG 139 ;; 140 --help|-h) 141 usage 142 ;; 143 --interactive) 144 TORTURE_QEMU_INTERACTIVE=1; export TORTURE_QEMU_INTERACTIVE 145 ;; 146 --jitter) 147 checkarg --jitter "(# threads [ sleep [ spin ] ])" $# "$2" '^-\{,1\}[0-9]\+\( \+[0-9]\+\)\{,2\} *$' '^error$' 148 jitter="$2" 149 shift 150 ;; 151 --kconfig) 152 checkarg --kconfig "(Kconfig options)" $# "$2" '^CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\)\( CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\)\)*$' '^error$' 153 TORTURE_KCONFIG_ARG="$2" 154 shift 155 ;; 156 --kasan) 157 TORTURE_KCONFIG_KASAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KASAN=y"; export TORTURE_KCONFIG_KASAN_ARG 158 ;; 159 --kcsan) 160 TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KCSAN=y CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY=n CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_VERBOSE=y CONFIG_KCSAN_INTERRUPT_WATCHER=y"; export TORTURE_KCONFIG_KCSAN_ARG 161 ;; 162 --kmake-arg) 163 checkarg --kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$' 164 TORTURE_KMAKE_ARG="$2" 165 shift 166 ;; 167 --mac) 168 checkarg --mac "(MAC address)" $# "$2" '^\([0-9a-fA-F]\{2\}:\)\{5\}[0-9a-fA-F]\{2\}$' error 169 TORTURE_QEMU_MAC=$2 170 shift 171 ;; 172 --memory) 173 checkarg --memory "(memory size)" $# "$2" '^[0-9]\+[MG]\?$' error 174 TORTURE_QEMU_MEM=$2 175 shift 176 ;; 177 --no-initrd) 178 TORTURE_INITRD=""; export TORTURE_INITRD 179 ;; 180 --qemu-args|--qemu-arg) 181 checkarg --qemu-args "(qemu arguments)" $# "$2" '^-' '^error' 182 TORTURE_QEMU_ARG="$2" 183 shift 184 ;; 185 --qemu-cmd) 186 checkarg --qemu-cmd "(qemu-system-...)" $# "$2" 'qemu-system-' '^--' 187 TORTURE_QEMU_CMD="$2" 188 shift 189 ;; 190 --results) 191 checkarg --results "(absolute pathname)" "$#" "$2" '^/' '^error' 192 resdir=$2 193 shift 194 ;; 195 --shutdown-grace) 196 checkarg --shutdown-grace "(seconds)" "$#" "$2" '^[0-9]*$' '^error' 197 TORTURE_SHUTDOWN_GRACE=$2 198 shift 199 ;; 200 --torture) 201 checkarg --torture "(suite name)" "$#" "$2" '^\(lock\|rcu\|rcuscale\|refscale\|scf\)$' '^--' 202 TORTURE_SUITE=$2 203 shift 204 if test "$TORTURE_SUITE" = rcuscale || test "$TORTURE_SUITE" = refscale 205 then 206 # If you really want jitter for refscale or 207 # rcuscale, specify it after specifying the rcuscale 208 # or the refscale. (But why jitter in these cases?) 209 jitter=0 210 fi 211 ;; 212 --trust-make) 213 TORTURE_TRUST_MAKE="y" 214 ;; 215 *) 216 echo Unknown argument $1 217 usage 218 ;; 219 esac 220 shift 221done 222 223if test -z "$TORTURE_INITRD" || tools/testing/selftests/rcutorture/bin/mkinitrd.sh 224then 225 : 226else 227 echo No initrd and unable to create one, aborting test >&2 228 exit 1 229fi 230 231CONFIGFRAG=${KVM}/configs/${TORTURE_SUITE}; export CONFIGFRAG 232 233defaultconfigs="`tr '\012' ' ' < $CONFIGFRAG/CFLIST`" 234if test -z "$configs" 235then 236 configs=$defaultconfigs 237fi 238 239if test -z "$resdir" 240then 241 resdir=$KVM/res 242fi 243 244# Create a file of test-name/#cpus pairs, sorted by decreasing #cpus. 245configs_derep= 246for CF in $configs 247do 248 case $CF in 249 [0-9]\**|[0-9][0-9]\**|[0-9][0-9][0-9]\**) 250 config_reps=`echo $CF | sed -e 's/\*.*$//'` 251 CF1=`echo $CF | sed -e 's/^[^*]*\*//'` 252 ;; 253 *) 254 config_reps=1 255 CF1=$CF 256 ;; 257 esac 258 for ((cur_rep=0;cur_rep<$config_reps;cur_rep++)) 259 do 260 configs_derep="$configs_derep $CF1" 261 done 262done 263touch $T/cfgcpu 264configs_derep="`echo $configs_derep | sed -e "s/\<CFLIST\>/$defaultconfigs/g"`" 265if test -n "$TORTURE_KCONFIG_GDB_ARG" 266then 267 if test "`echo $configs_derep | wc -w`" -gt 1 268 then 269 echo "The --config list is: $configs_derep." 270 echo "Only one --config permitted with --gdb, terminating." 271 exit 1 272 fi 273fi 274for CF1 in $configs_derep 275do 276 if test -f "$CONFIGFRAG/$CF1" 277 then 278 cpu_count=`configNR_CPUS.sh $CONFIGFRAG/$CF1` 279 cpu_count=`configfrag_boot_cpus "$TORTURE_BOOTARGS" "$CONFIGFRAG/$CF1" "$cpu_count"` 280 cpu_count=`configfrag_boot_maxcpus "$TORTURE_BOOTARGS" "$CONFIGFRAG/$CF1" "$cpu_count"` 281 echo $CF1 $cpu_count >> $T/cfgcpu 282 else 283 echo "The --configs file $CF1 does not exist, terminating." 284 exit 1 285 fi 286done 287sort -k2nr $T/cfgcpu -T="$T" > $T/cfgcpu.sort 288 289# Use a greedy bin-packing algorithm, sorting the list accordingly. 290awk < $T/cfgcpu.sort > $T/cfgcpu.pack -v ncpus=$cpus ' 291BEGIN { 292 njobs = 0; 293} 294 295{ 296 # Read file of tests and corresponding required numbers of CPUs. 297 cf[njobs] = $1; 298 cpus[njobs] = $2; 299 njobs++; 300} 301 302END { 303 batch = 0; 304 nc = -1; 305 306 # Each pass through the following loop creates on test batch 307 # that can be executed concurrently given ncpus. Note that a 308 # given test that requires more than the available CPUs will run in 309 # their own batch. Such tests just have to make do with what 310 # is available. 311 while (nc != ncpus) { 312 batch++; 313 nc = ncpus; 314 315 # Each pass through the following loop considers one 316 # test for inclusion in the current batch. 317 for (i = 0; i < njobs; i++) { 318 if (done[i]) 319 continue; # Already part of a batch. 320 if (nc >= cpus[i] || nc == ncpus) { 321 322 # This test fits into the current batch. 323 done[i] = batch; 324 nc -= cpus[i]; 325 if (nc <= 0) 326 break; # Too-big test in its own batch. 327 } 328 } 329 } 330 331 # Dump out the tests in batch order. 332 for (b = 1; b <= batch; b++) 333 for (i = 0; i < njobs; i++) 334 if (done[i] == b) 335 print cf[i], cpus[i]; 336}' 337 338# Generate a script to execute the tests in appropriate batches. 339cat << ___EOF___ > $T/script 340CONFIGFRAG="$CONFIGFRAG"; export CONFIGFRAG 341KVM="$KVM"; export KVM 342PATH="$PATH"; export PATH 343TORTURE_ALLOTED_CPUS="$TORTURE_ALLOTED_CPUS"; export TORTURE_ALLOTED_CPUS 344TORTURE_BOOT_IMAGE="$TORTURE_BOOT_IMAGE"; export TORTURE_BOOT_IMAGE 345TORTURE_BUILDONLY="$TORTURE_BUILDONLY"; export TORTURE_BUILDONLY 346TORTURE_DEFCONFIG="$TORTURE_DEFCONFIG"; export TORTURE_DEFCONFIG 347TORTURE_INITRD="$TORTURE_INITRD"; export TORTURE_INITRD 348TORTURE_KCONFIG_ARG="$TORTURE_KCONFIG_ARG"; export TORTURE_KCONFIG_ARG 349TORTURE_KCONFIG_GDB_ARG="$TORTURE_KCONFIG_GDB_ARG"; export TORTURE_KCONFIG_GDB_ARG 350TORTURE_BOOT_GDB_ARG="$TORTURE_BOOT_GDB_ARG"; export TORTURE_BOOT_GDB_ARG 351TORTURE_QEMU_GDB_ARG="$TORTURE_QEMU_GDB_ARG"; export TORTURE_QEMU_GDB_ARG 352TORTURE_KCONFIG_KASAN_ARG="$TORTURE_KCONFIG_KASAN_ARG"; export TORTURE_KCONFIG_KASAN_ARG 353TORTURE_KCONFIG_KCSAN_ARG="$TORTURE_KCONFIG_KCSAN_ARG"; export TORTURE_KCONFIG_KCSAN_ARG 354TORTURE_KMAKE_ARG="$TORTURE_KMAKE_ARG"; export TORTURE_KMAKE_ARG 355TORTURE_QEMU_CMD="$TORTURE_QEMU_CMD"; export TORTURE_QEMU_CMD 356TORTURE_QEMU_INTERACTIVE="$TORTURE_QEMU_INTERACTIVE"; export TORTURE_QEMU_INTERACTIVE 357TORTURE_QEMU_MAC="$TORTURE_QEMU_MAC"; export TORTURE_QEMU_MAC 358TORTURE_QEMU_MEM="$TORTURE_QEMU_MEM"; export TORTURE_QEMU_MEM 359TORTURE_SHUTDOWN_GRACE="$TORTURE_SHUTDOWN_GRACE"; export TORTURE_SHUTDOWN_GRACE 360TORTURE_SUITE="$TORTURE_SUITE"; export TORTURE_SUITE 361TORTURE_TRUST_MAKE="$TORTURE_TRUST_MAKE"; export TORTURE_TRUST_MAKE 362if ! test -e $resdir 363then 364 mkdir -p "$resdir" || : 365fi 366mkdir $resdir/$ds 367TORTURE_RESDIR="$resdir/$ds"; export TORTURE_RESDIR 368TORTURE_STOPFILE="$resdir/$ds/STOP"; export TORTURE_STOPFILE 369echo Results directory: $resdir/$ds 370echo $scriptname $args 371touch $resdir/$ds/log 372echo $scriptname $args >> $resdir/$ds/log 373echo ${TORTURE_SUITE} > $resdir/$ds/TORTURE_SUITE 374pwd > $resdir/$ds/testid.txt 375if test -d .git 376then 377 git status >> $resdir/$ds/testid.txt 378 git rev-parse HEAD >> $resdir/$ds/testid.txt 379 git diff HEAD >> $resdir/$ds/testid.txt 380fi 381___EOF___ 382awk < $T/cfgcpu.pack \ 383 -v TORTURE_BUILDONLY="$TORTURE_BUILDONLY" \ 384 -v CONFIGDIR="$CONFIGFRAG/" \ 385 -v KVM="$KVM" \ 386 -v ncpus=$cpus \ 387 -v jitter="$jitter" \ 388 -v rd=$resdir/$ds/ \ 389 -v dur=$dur \ 390 -v TORTURE_QEMU_ARG="$TORTURE_QEMU_ARG" \ 391 -v TORTURE_BOOTARGS="$TORTURE_BOOTARGS" \ 392'BEGIN { 393 i = 0; 394} 395 396{ 397 cf[i] = $1; 398 cpus[i] = $2; 399 i++; 400} 401 402# Dump out the scripting required to run one test batch. 403function dump(first, pastlast, batchnum) 404{ 405 print "echo ----Start batch " batchnum ": `date` | tee -a " rd "log"; 406 print "needqemurun=" 407 jn=1 408 for (j = first; j < pastlast; j++) { 409 builddir=KVM "/b" j - first + 1 410 cpusr[jn] = cpus[j]; 411 if (cfrep[cf[j]] == "") { 412 cfr[jn] = cf[j]; 413 cfrep[cf[j]] = 1; 414 } else { 415 cfrep[cf[j]]++; 416 cfr[jn] = cf[j] "." cfrep[cf[j]]; 417 } 418 if (cpusr[jn] > ncpus && ncpus != 0) 419 ovf = "-ovf"; 420 else 421 ovf = ""; 422 print "echo ", cfr[jn], cpusr[jn] ovf ": Starting build. `date` | tee -a " rd "log"; 423 print "rm -f " builddir ".*"; 424 print "touch " builddir ".wait"; 425 print "mkdir " rd cfr[jn] " || :"; 426 print "kvm-test-1-run.sh " CONFIGDIR cf[j], builddir, rd cfr[jn], dur " \"" TORTURE_QEMU_ARG "\" \"" TORTURE_BOOTARGS "\" > " rd cfr[jn] "/kvm-test-1-run.sh.out 2>&1 &" 427 print "echo ", cfr[jn], cpusr[jn] ovf ": Waiting for build to complete. `date` | tee -a " rd "log"; 428 print "while test -f " builddir ".wait" 429 print "do" 430 print "\tsleep 1" 431 print "done" 432 print "echo ", cfr[jn], cpusr[jn] ovf ": Build complete. `date` | tee -a " rd "log"; 433 jn++; 434 } 435 for (j = 1; j < jn; j++) { 436 builddir=KVM "/b" j 437 print "rm -f " builddir ".ready" 438 print "if test -f \"" rd cfr[j] "/builtkernel\"" 439 print "then" 440 print "\techo ----", cfr[j], cpusr[j] ovf ": Kernel present. `date` | tee -a " rd "log"; 441 print "\tneedqemurun=1" 442 print "fi" 443 } 444 njitter = 0; 445 split(jitter, ja); 446 if (ja[1] == -1 && ncpus == 0) 447 njitter = 1; 448 else if (ja[1] == -1) 449 njitter = ncpus; 450 else 451 njitter = ja[1]; 452 if (TORTURE_BUILDONLY && njitter != 0) { 453 njitter = 0; 454 print "echo Build-only run, so suppressing jitter | tee -a " rd "log" 455 } 456 if (TORTURE_BUILDONLY) { 457 print "needqemurun=" 458 } 459 print "if test -n \"$needqemurun\"" 460 print "then" 461 print "\techo ---- Starting kernels. `date` | tee -a " rd "log"; 462 for (j = 0; j < njitter; j++) 463 print "\tjitter.sh " j " " dur " " ja[2] " " ja[3] "&" 464 print "\twait" 465 print "\techo ---- All kernel runs complete. `date` | tee -a " rd "log"; 466 print "else" 467 print "\twait" 468 print "\techo ---- No kernel runs. `date` | tee -a " rd "log"; 469 print "fi" 470 for (j = 1; j < jn; j++) { 471 builddir=KVM "/b" j 472 print "echo ----", cfr[j], cpusr[j] ovf ": Build/run results: | tee -a " rd "log"; 473 print "cat " rd cfr[j] "/kvm-test-1-run.sh.out | tee -a " rd "log"; 474 } 475} 476 477END { 478 njobs = i; 479 nc = ncpus; 480 first = 0; 481 batchnum = 1; 482 483 # Each pass through the following loop considers one test. 484 for (i = 0; i < njobs; i++) { 485 if (ncpus == 0) { 486 # Sequential test specified, each test its own batch. 487 dump(i, i + 1, batchnum); 488 first = i; 489 batchnum++; 490 } else if (nc < cpus[i] && i != 0) { 491 # Out of CPUs, dump out a batch. 492 dump(first, i, batchnum); 493 first = i; 494 nc = ncpus; 495 batchnum++; 496 } 497 # Account for the CPUs needed by the current test. 498 nc -= cpus[i]; 499 } 500 # Dump the last batch. 501 if (ncpus != 0) 502 dump(first, i, batchnum); 503}' >> $T/script 504 505cat << ___EOF___ >> $T/script 506echo 507echo 508echo " --- `date` Test summary:" 509echo Results directory: $resdir/$ds 510kcsan-collapse.sh $resdir/$ds 511kvm-recheck.sh $resdir/$ds 512___EOF___ 513 514if test "$dryrun" = script 515then 516 cat $T/script 517 exit 0 518elif test "$dryrun" = sched 519then 520 # Extract the test run schedule from the script. 521 egrep 'Start batch|Starting build\.' $T/script | 522 grep -v ">>" | 523 sed -e 's/:.*$//' -e 's/^echo //' 524 exit 0 525else 526 # Not a dryrun, so run the script. 527 sh $T/script 528fi 529 530# Tracing: trace_event=rcu:rcu_grace_period,rcu:rcu_future_grace_period,rcu:rcu_grace_period_init,rcu:rcu_nocb_wake,rcu:rcu_preempt_task,rcu:rcu_unlock_preempted_task,rcu:rcu_quiescent_state_report,rcu:rcu_fqs,rcu:rcu_callback,rcu:rcu_kfree_callback,rcu:rcu_batch_start,rcu:rcu_invoke_callback,rcu:rcu_invoke_kfree_callback,rcu:rcu_batch_end,rcu:rcu_torture_read,rcu:rcu_barrier 531# Function-graph tracing: ftrace=function_graph ftrace_graph_filter=sched_setaffinity,migration_cpu_stop 532# Also --kconfig "CONFIG_FUNCTION_TRACER=y CONFIG_FUNCTION_GRAPH_TRACER=y" 533# Control buffer size: --bootargs trace_buf_size=3k 534# Get trace-buffer dumps on all oopses: --bootargs ftrace_dump_on_oops 535# Ditto, but dump only the oopsing CPU: --bootargs ftrace_dump_on_oops=orig_cpu 536# Heavy-handed way to also dump on warnings: --bootargs panic_on_warn 537