1#!/bin/bash 2usage() { 3echo " 4 Run end-to-end tests in parallel. 5 6 Usage: 7 ./regtest.sh <function name> 8 At the end, it will print an HTML summary. 9 10 Three main functions are 11 run [<pattern> [<lang>]] - run tests matching <pattern> in 12 parallel. The language 13 of the client to use. 14 run-seq [<pattern> [<lang>]] - ditto, except that tests are run 15 sequentially 16 run-all - run all tests, in parallel 17 18 Examples: 19 $ ./regtest.sh run-seq unif-small-typical # Run, the unif-small-typical test 20 $ ./regtest.sh run-seq unif-small- # Sequential, the tests containing: 21 # 'unif-small-' 22 $ ./regtest.sh run unif- # Parallel run, matches multiple cases 23 $ ./regtest.sh run-all # Run all tests 24 25 The <pattern> argument is a regex in 'grep -E' format. (Detail: Don't 26 use $ in the pattern, since it matches the whole spec line and not just the 27 test case name.) The number of processors used in a parallel run is one less 28 than the number of CPUs on the machine. 29" 30} 31# Future speedups: 32# - Reuse the same input -- come up with naming scheme based on params 33# - Reuse the same maps -- ditto, rappor library can cache it 34# 35 36set -o nounset 37set -o pipefail 38set -o errexit 39 40. util.sh 41 42readonly THIS_DIR=$(dirname $0) 43readonly REPO_ROOT=$THIS_DIR 44readonly CLIENT_DIR=$REPO_ROOT/client/python 45# subdirs are in _tmp/$impl, which shouldn't overlap with anything else in _tmp 46readonly REGTEST_BASE_DIR=_tmp 47 48# All the Python tools need this 49export PYTHONPATH=$CLIENT_DIR 50 51print-unique-values() { 52 local num_unique_values=$1 53 seq 1 $num_unique_values | awk '{print "v" $1}' 54} 55 56# Add some more candidates here. We hope these are estimated at 0. 57# e.g. if add_start=51, and num_additional is 20, show v51-v70 58more-candidates() { 59 local last_true=$1 60 local num_additional=$2 61 62 local begin 63 local end 64 begin=$(expr $last_true + 1) 65 end=$(expr $last_true + $num_additional) 66 67 seq $begin $end | awk '{print "v" $1}' 68} 69 70# Args: 71# unique_values: File of unique true values 72# last_true: last true input, e.g. 50 if we generated "v1" .. "v50". 73# num_additional: additional candidates to generate (starting at 'last_true') 74# to_remove: Regex of true values to omit from the candidates list, or the 75# string 'NONE' if none should be. (Our values look like 'v1', 'v2', etc. so 76# there isn't any ambiguity.) 77print-candidates() { 78 local unique_values=$1 79 local last_true=$2 80 local num_additional=$3 81 local to_remove=$4 82 83 if test $to_remove = NONE; then 84 cat $unique_values # include all true inputs 85 else 86 egrep -v $to_remove $unique_values # remove some true inputs 87 fi 88 more-candidates $last_true $num_additional 89} 90 91# Generate a single test case, specified by a line of the test spec. 92# This is a helper function for _run_tests(). 93_setup-one-case() { 94 local impl=$1 95 shift # impl is not part of the spec; the next 13 params are 96 97 local test_case=$1 98 99 # input params 100 local dist=$2 101 local num_unique_values=$3 102 local num_clients=$4 103 local values_per_client=$5 104 105 # RAPPOR params 106 local num_bits=$6 107 local num_hashes=$7 108 local num_cohorts=$8 109 local p=$9 110 local q=${10} # need curly braces to get the 10th arg 111 local f=${11} 112 113 # map params 114 local num_additional=${12} 115 local to_remove=${13} 116 117 banner 'Setting up parameters and candidate files for '$test_case 118 119 local case_dir=$REGTEST_BASE_DIR/$impl/$test_case 120 mkdir --verbose -p $case_dir 121 122 # Save the "spec" 123 echo "$@" > $case_dir/spec.txt 124 125 local params_path=$case_dir/case_params.csv 126 127 echo 'k,h,m,p,q,f' > $params_path 128 echo "$num_bits,$num_hashes,$num_cohorts,$p,$q,$f" >> $params_path 129 130 print-unique-values $num_unique_values > $case_dir/case_unique_values.txt 131 132 local true_map_path=$case_dir/case_true_map.csv 133 134 bin/hash_candidates.py \ 135 $params_path \ 136 < $case_dir/case_unique_values.txt \ 137 > $true_map_path 138 139 # banner "Constructing candidates" 140 141 print-candidates \ 142 $case_dir/case_unique_values.txt $num_unique_values \ 143 $num_additional "$to_remove" \ 144 > $case_dir/case_candidates.txt 145 146 # banner "Hashing candidates to get 'map'" 147 148 bin/hash_candidates.py \ 149 $params_path \ 150 < $case_dir/case_candidates.txt \ 151 > $case_dir/case_map.csv 152} 153 154# Run a single test instance, specified by <test_name, instance_num>. 155# This is a helper function for _run_tests(). 156_run-one-instance() { 157 local test_case=$1 158 local test_instance=$2 159 local impl=$3 160 161 local case_dir=$REGTEST_BASE_DIR/$impl/$test_case 162 163 read -r \ 164 case_name distr num_unique_values num_clients values_per_client \ 165 num_bits num_hashes num_cohorts p q f \ 166 num_additional to_remove \ 167 < $case_dir/spec.txt 168 169 local instance_dir=$case_dir/$test_instance 170 mkdir --verbose -p $instance_dir 171 172 banner "Generating reports (gen_reports.R)" 173 174 # the TRUE_VALUES_PATH environment variable can be used to avoid 175 # generating new values every time. NOTE: You are responsible for making 176 # sure the params match! 177 178 local true_values=${TRUE_VALUES_PATH:-} 179 if test -z "$true_values"; then 180 true_values=$instance_dir/case_true_values.csv 181 tests/gen_true_values.R $distr $num_unique_values $num_clients \ 182 $values_per_client $num_cohorts \ 183 $true_values 184 else 185 # TEMP hack: Make it visible to plot. 186 # TODO: Fix compare_dist.R 187 ln -s -f --verbose \ 188 $PWD/$true_values \ 189 $instance_dir/case_true_values.csv 190 fi 191 192 case $impl in 193 python) 194 banner "Running RAPPOR Python client" 195 196 # Writes encoded "out" file, true histogram, true inputs to 197 # $instance_dir. 198 time tests/rappor_sim.py \ 199 --num-bits $num_bits \ 200 --num-hashes $num_hashes \ 201 --num-cohorts $num_cohorts \ 202 -p $p \ 203 -q $q \ 204 -f $f \ 205 < $true_values \ 206 > "$instance_dir/case_reports.csv" 207 ;; 208 209 cpp) 210 banner "Running RAPPOR C++ client (see rappor_sim.log for errors)" 211 212 time client/cpp/_tmp/rappor_sim \ 213 $num_bits \ 214 $num_hashes \ 215 $num_cohorts \ 216 $p \ 217 $q \ 218 $f \ 219 < $true_values \ 220 > "$instance_dir/case_reports.csv" \ 221 2>"$instance_dir/rappor_sim.log" 222 ;; 223 224 *) 225 log "Invalid impl $impl (should be one of python|cpp)" 226 exit 1 227 ;; 228 229 esac 230 231 banner "Summing RAPPOR IRR bits to get 'counts'" 232 233 bin/sum_bits.py \ 234 $case_dir/case_params.csv \ 235 < $instance_dir/case_reports.csv \ 236 > $instance_dir/case_counts.csv 237 238 local out_dir=${instance_dir}_report 239 mkdir --verbose -p $out_dir 240 241 # Currently, the summary file shows and aggregates timing of the inference 242 # engine, which excludes R's loading time and reading of the (possibly 243 # substantial) map file. Timing below is more inclusive. 244 TIMEFORMAT='Running compare_dist.R took %R seconds' 245 time { 246 # Input prefix, output dir 247 tests/compare_dist.R -t "Test case: $test_case (instance $test_instance)" \ 248 "$case_dir/case" "$instance_dir/case" $out_dir 249 } 250} 251 252# Like _run-once-case, but log to a file. 253_run-one-instance-logged() { 254 local test_case=$1 255 local test_instance=$2 256 local impl=$3 257 258 local log_dir=$REGTEST_BASE_DIR/$impl/$test_case/${test_instance}_report 259 mkdir --verbose -p $log_dir 260 261 log "Started '$test_case' (instance $test_instance) -- logging to $log_dir/log.txt" 262 _run-one-instance "$@" >$log_dir/log.txt 2>&1 \ 263 && log "Test case $test_case (instance $test_instance) done" \ 264 || log "Test case $test_case (instance $test_instance) failed" 265} 266 267make-summary() { 268 local dir=$1 269 local impl=$2 270 271 local filename=results.html 272 273 tests/make_summary.py $dir $dir/rows.html 274 275 pushd $dir >/dev/null 276 277 cat ../../tests/regtest.html \ 278 | sed -e '/__TABLE_ROWS__/ r rows.html' -e "s/_IMPL_/$impl/g" \ 279 > $filename 280 281 popd >/dev/null 282 283 log "Wrote $dir/$filename" 284 log "URL: file://$PWD/$dir/$filename" 285} 286 287test-error() { 288 local spec_regex=${1:-} 289 log "Some test cases failed" 290 if test -n "$spec_regex"; then 291 log "(Perhaps none matched pattern '$spec_regex')" 292 fi 293 # don't quit just yet 294 # exit 1 295} 296 297# Assuming the spec file, write a list of test case names (first column) with 298# the instance ids (second column), where instance ids run from 1 to $1. 299# Third column is impl. 300_setup-test-instances() { 301 local instances=$1 302 local impl=$2 303 304 while read line; do 305 for i in $(seq 1 $instances); do 306 read case_name _ <<< $line # extract the first token 307 echo $case_name $i $impl 308 done 309 done 310} 311 312# Print the default number of parallel processes, which is max(#CPUs - 1, 1) 313default-processes() { 314 processors=$(grep -c ^processor /proc/cpuinfo || echo 4) # Linux-specific 315 if test $processors -gt 1; then # leave one CPU for the OS 316 processors=$(expr $processors - 1) 317 fi 318 echo $processors 319} 320 321# Args: 322# spec_gen: A program to execute to generate the spec. 323# spec_regex: A pattern selecting the subset of tests to run 324# parallel: Whether the tests are run in parallel (T/F). Sequential 325# runs log to the console; parallel runs log to files. 326# impl: one of python, or cpp 327# instances: A number of times each test case is run 328 329_run-tests() { 330 local spec_gen=$1 331 local spec_regex="$2" # grep -E format on the spec, can be empty 332 local parallel=$3 333 local impl=${4:-"cpp"} 334 local instances=${5:-1} 335 336 local regtest_dir=$REGTEST_BASE_DIR/$impl 337 rm -r -f --verbose $regtest_dir 338 339 mkdir --verbose -p $regtest_dir 340 341 local func 342 local processors 343 344 if test $parallel = F; then 345 func=_run-one-instance # output to the console 346 processors=1 347 else 348 func=_run-one-instance-logged 349 # Let the user override with MAX_PROC, in case they don't have enough 350 # memory. 351 processors=${MAX_PROC:-$(default-processes)} 352 log "Running $processors parallel processes" 353 fi 354 355 local cases_list=$regtest_dir/test-cases.txt 356 # Need -- for regexes that start with - 357 $spec_gen | grep -E -- "$spec_regex" > $cases_list 358 359 # Generate parameters for all test cases. 360 cat $cases_list \ 361 | xargs -l -P $processors -- $0 _setup-one-case $impl \ 362 || test-error 363 364 log "Done generating parameters for all test cases" 365 366 local instances_list=$regtest_dir/test-instances.txt 367 _setup-test-instances $instances $impl < $cases_list > $instances_list 368 369 cat $instances_list \ 370 | xargs -l -P $processors -- $0 $func || test-error 371 372 log "Done running all test instances" 373 374 make-summary $regtest_dir $impl 375} 376 377# used for most tests 378readonly REGTEST_SPEC=tests/regtest_spec.py 379 380# Run tests sequentially. NOTE: called by demo.sh. 381run-seq() { 382 local spec_regex=${1:-'^r-'} # grep -E format on the spec 383 shift 384 385 time _run-tests $REGTEST_SPEC $spec_regex F $@ 386} 387 388# Run tests in parallel 389run() { 390 local spec_regex=${1:-'^r-'} # grep -E format on the spec 391 shift 392 393 time _run-tests $REGTEST_SPEC $spec_regex T $@ 394} 395 396# Run tests in parallel (7+ minutes on 8 cores) 397run-all() { 398 log "Running all tests. Can take a while." 399 time _run-tests $REGTEST_SPEC '^r-' T cpp 400} 401 402run-user() { 403 local spec_regex=${1:-} 404 local parallel=T # too much memory 405 time _run-tests tests/user_spec.py "$spec_regex" $parallel cpp 406} 407 408# Use stable true values 409compare-python-cpp() { 410 local num_unique_values=100 411 local num_clients=10000 412 local values_per_client=10 413 local num_cohorts=64 414 415 local true_values=$REGTEST_BASE_DIR/stable_true_values.csv 416 417 tests/gen_true_values.R \ 418 exp $num_unique_values $num_clients $values_per_client $num_cohorts \ 419 $true_values 420 421 wc -l $true_values 422 423 # Run Python and C++ simulation on the same input 424 425 ./build.sh cpp-client 426 427 TRUE_VALUES_PATH=$true_values \ 428 ./regtest.sh run-seq '^demo3' 1 python 429 430 TRUE_VALUES_PATH=$true_values \ 431 ./regtest.sh run-seq '^demo3' 1 cpp 432 433 head _tmp/{python,cpp}/demo3/1/case_reports.csv 434} 435 436if test $# -eq 0 ; then 437 usage 438else 439 "$@" 440fi 441