1#!/bin/bash 2# Copyright 2019 Huawei Technologies Co., Ltd 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# ============================================================================ 16 17# source the globals and functions for use with cache testing 18export SKIP_ADMIN_COUNTER=true 19declare session_id failed_tests 20. cachetest_lib.sh 21echo 22 23################################################################################ 24# Cache testing: cache python test driver # 25# Summary: Various tests for running the python testcases for caching # 26################################################################################ 27 28StartServer 29HandleRcExit $? 1 1 30 31# Set the environment variable to enable these pytests 32export RUN_CACHE_TEST=TRUE 33 34# Each of these tests will create session, use it, then destroy it after the test 35for i in $(seq 1 5) 36do 37 test_name="test_cache_map_basic${i}" 38 GetSession 39 HandleRcExit $? 1 1 40 export SESSION_ID=$session_id 41 42 PytestCmd "test_cache_map.py" "${test_name}" 43 HandleRcExit $? 0 0 44 45 DestroySession $session_id 46 HandleRcExit $? 1 1 47done 48 49GetSession 50HandleRcExit $? 1 1 51export SESSION_ID=$session_id 52 53# use pytest pattern match to run all the tests that match the name test_cache_map_failure. 54# All of these tests will interact with the same cache session and may result in multiple 55# caches under the common session handle (although these are failure tests so probably not) 56PytestCmd "test_cache_map.py" "test_cache_map_failure" 1 57HandleRcExit $? 0 0 58 59PytestCmd "test_cache_map.py" "test_cache_map_split" 1 60HandleRcExit $? 0 0 61 62# DatasetCache parameter check 63PytestCmd "test_cache_map.py" "test_cache_map_parameter_check" 64HandleRcExit $? 0 0 65 66# Executing the same pipeline for twice under the same session 67# Executing the same pipeline for twice (from python) 68PytestCmd "test_cache_map.py" "test_cache_map_running_twice1" 69HandleRcExit $? 0 0 70# Executing the same pipeline for twice (from shell) 71PytestCmd "test_cache_map.py" "test_cache_map_running_twice2" 72HandleRcExit $? 0 0 73PytestCmd "test_cache_map.py" "test_cache_map_running_twice2" 74HandleRcExit $? 0 0 75 76# Executing the same pipeline for twice under the different session 77# Executing the same pipeline for twice (from shell) 78PytestCmd "test_cache_map.py" "test_cache_map_running_twice2" 79HandleRcExit $? 0 0 80DestroySession $session_id 81HandleRcExit $? 1 1 82GetSession 83HandleRcExit $? 1 1 84export SESSION_ID=$session_id 85PytestCmd "test_cache_map.py" "test_cache_map_running_twice2" 86HandleRcExit $? 0 0 87 88PytestCmd "test_cache_map.py" "test_cache_map_no_image" 89HandleRcExit $? 0 0 90 91PytestCmd "test_cache_map.py" "test_cache_map_parallel_workers" 92HandleRcExit $? 0 0 93 94PytestCmd "test_cache_map.py" "test_cache_map_num_connections" 1 95HandleRcExit $? 0 0 96 97PytestCmd "test_cache_map.py" "test_cache_map_prefetch_size" 1 98HandleRcExit $? 0 0 99 100PytestCmd "test_cache_map.py" "test_cache_map_to_device" 101HandleRcExit $? 0 0 102 103PytestCmd "test_cache_map.py" "test_cache_map_epoch_ctrl" 1 104HandleRcExit $? 0 0 105 106PytestCmd "test_cache_map.py" "test_cache_map_coco" 1 107HandleRcExit $? 0 0 108 109PytestCmd "test_cache_map.py" "test_cache_map_mnist" 1 110HandleRcExit $? 0 0 111 112PytestCmd "test_cache_map.py" "test_cache_map_celeba" 1 113HandleRcExit $? 0 0 114 115PytestCmd "test_cache_map.py" "test_cache_map_manifest" 1 116HandleRcExit $? 0 0 117 118PytestCmd "test_cache_map.py" "test_cache_map_cifar" 1 119HandleRcExit $? 0 0 120 121PytestCmd "test_cache_map.py" "test_cache_map_voc" 1 122HandleRcExit $? 0 0 123 124PytestCmd "test_cache_map.py" "test_cache_map_mindrecord" 1 125HandleRcExit $? 0 0 126 127PytestCmd "test_cache_map.py" "test_cache_map_python_sampler" 1 128HandleRcExit $? 0 0 129 130PytestCmd "test_cache_map.py" "test_cache_map_nested_repeat" 131HandleRcExit $? 0 0 132 133PytestCmd "test_cache_map.py" "test_cache_map_dataset_size" 1 134HandleRcExit $? 0 0 135 136GetSession 137HandleRcExit $? 1 1 138export SESSION_ID=$session_id 139 140PytestCmd "test_cache_map.py" "test_cache_map_interrupt_and_rerun" 141HandleRcExit $? 0 0 142 143DestroySession $session_id 144HandleRcExit $? 1 1 145 146# Run two parallel pipelines (sharing cache) 147for i in $(seq 1 2) 148do 149 test_name="test_cache_map_parallel_pipeline${i}" 150 GetSession 151 HandleRcExit $? 1 1 152 export SESSION_ID=$session_id 153 154 PytestCmd "test_cache_map.py" "${test_name} --shard 0" & 155 pids+=("$!") 156 PytestCmd "test_cache_map.py" "${test_name} --shard 1" & 157 pids+=("$!") 158 159 for pid in "${pids[@]}"; do 160 wait ${pid} 161 HandleRcExit $? 0 0 162 done 163 164 # Running those PytestCmd in the background will not get our test_count updated. So we need to manually update it here. 165 test_count=$(($test_count+1)) 166 DestroySession $session_id 167 HandleRcExit $? 1 1 168done 169 170StopServer 171HandleRcExit $? 1 1 172sleep 1 173 174# test cache server with --workers 1 175cmd="${CACHE_ADMIN} --start --workers 1" 176CacheAdminCmd "${cmd}" 0 177sleep 1 178HandleRcExit $? 0 0 179 180GetSession 181HandleRcExit $? 1 1 182export SESSION_ID=$session_id 183 184PytestCmd "test_cache_map.py" "test_cache_map_server_workers_1" 185HandleRcExit $? 0 0 186StopServer 187HandleRcExit $? 0 1 188 189# test cache server with --workers 100 190cmd="${CACHE_ADMIN} --start --workers 100" 191CacheAdminCmd "${cmd}" 0 192sleep 1 193HandleRcExit $? 0 0 194 195GetSession 196HandleRcExit $? 1 1 197export SESSION_ID=$session_id 198 199PytestCmd "test_cache_map.py" "test_cache_map_server_workers_100" 200HandleRcExit $? 0 0 201StopServer 202HandleRcExit $? 0 1 203 204# The next set of testing is for the non-mappable cases. 205StartServer 206HandleRcExit $? 1 1 207 208# This runs all of the basic tests. These will all share the same and we do not destroy 209# the session in between each. 210GetSession 211HandleRcExit $? 1 1 212export SESSION_ID=$session_id 213 214PytestCmd "test_cache_nomap.py" "test_cache_nomap_basic" 1 215HandleRcExit $? 0 0 216 217DestroySession $session_id 218HandleRcExit $? 1 1 219 220# run the small shared cache tests 221for i in $(seq 1 4) 222do 223 test_name="test_cache_nomap_allowed_share${i}" 224 GetSession 225 HandleRcExit $? 1 1 226 export SESSION_ID=$session_id 227 228 PytestCmd "test_cache_nomap.py" "${test_name}" 229 HandleRcExit $? 0 0 230 231 DestroySession $session_id 232 HandleRcExit $? 1 1 233done 234 235GetSession 236HandleRcExit $? 1 1 237export SESSION_ID=$session_id 238 239PytestCmd "test_cache_nomap.py" "test_cache_nomap_disallowed_share" 1 240HandleRcExit $? 0 0 241 242DestroySession $session_id 243HandleRcExit $? 1 1 244 245GetSession 246HandleRcExit $? 1 1 247export SESSION_ID=$session_id 248 249# Executing the same pipeline for twice under the same session 250# Executing the same pipeline for twice (from python) 251PytestCmd "test_cache_nomap.py" "test_cache_nomap_running_twice1" 252HandleRcExit $? 0 0 253# Executing the same pipeline for twice (from shell) 254PytestCmd "test_cache_nomap.py" "test_cache_nomap_running_twice2" 255HandleRcExit $? 0 0 256PytestCmd "test_cache_nomap.py" "test_cache_nomap_running_twice2" 257HandleRcExit $? 0 0 258 259# Executing the same pipeline for twice under the different session 260# Executing the same pipeline for twice (from shell) 261PytestCmd "test_cache_nomap.py" "test_cache_nomap_running_twice2" 262HandleRcExit $? 0 0 263DestroySession $session_id 264HandleRcExit $? 1 1 265GetSession 266HandleRcExit $? 1 1 267export SESSION_ID=$session_id 268PytestCmd "test_cache_nomap.py" "test_cache_nomap_running_twice2" 269HandleRcExit $? 0 0 270 271# Run two parallel pipelines (sharing cache) 272for i in $(seq 1 2) 273do 274 test_name="test_cache_nomap_parallel_pipeline${i}" 275 GetSession 276 HandleRcExit $? 1 1 277 export SESSION_ID=$session_id 278 279 PytestCmd "test_cache_nomap.py" "${test_name} --shard 0" & 280 pids+=("$!") 281 PytestCmd "test_cache_nomap.py" "${test_name} --shard 1" & 282 pids+=("$!") 283 PytestCmd "test_cache_nomap.py" "${test_name} --shard 2" & 284 pids+=("$!") 285 286 for pid in "${pids[@]}"; do 287 wait ${pid} 288 HandleRcExit $? 0 0 289 done 290 291 # Running those PytestCmd in the background will not get our test_count updated. So we need to manually update it here. 292 test_count=$(($test_count+1)) 293 DestroySession $session_id 294 HandleRcExit $? 1 1 295done 296 297GetSession 298HandleRcExit $? 1 1 299export SESSION_ID=$session_id 300 301PytestCmd "test_cache_nomap.py" "test_cache_nomap_parallel_workers" 302HandleRcExit $? 0 0 303 304PytestCmd "test_cache_nomap.py" "test_cache_nomap_num_connections" 1 305HandleRcExit $? 0 0 306 307PytestCmd "test_cache_nomap.py" "test_cache_nomap_prefetch_size" 1 308HandleRcExit $? 0 0 309 310PytestCmd "test_cache_nomap.py" "test_cache_nomap_to_device" 311HandleRcExit $? 0 0 312 313PytestCmd "test_cache_nomap.py" "test_cache_nomap_epoch_ctrl" 1 314HandleRcExit $? 0 0 315 316PytestCmd "test_cache_nomap.py" "test_cache_nomap_clue" 1 317HandleRcExit $? 0 0 318 319PytestCmd "test_cache_nomap.py" "test_cache_nomap_csv" 1 320HandleRcExit $? 0 0 321 322PytestCmd "test_cache_nomap.py" "test_cache_nomap_textfile" 1 323HandleRcExit $? 0 0 324 325PytestCmd "test_cache_nomap.py" "test_cache_nomap_nested_repeat" 326HandleRcExit $? 0 0 327 328PytestCmd "test_cache_nomap.py" "test_cache_nomap_get_repeat_count" 329HandleRcExit $? 0 0 330 331PytestCmd "test_cache_nomap.py" "test_cache_nomap_long_file_list" 332HandleRcExit $? 0 0 333 334PytestCmd "test_cache_nomap.py" "test_cache_nomap_failure" 1 335HandleRcExit $? 0 0 336 337PytestCmd "test_cache_nomap.py" "test_cache_nomap_pyfunc" 1 338HandleRcExit $? 0 0 339 340PytestCmd "test_cache_nomap.py" "test_cache_nomap_dataset_size" 1 341HandleRcExit $? 0 0 342 343GetSession 344HandleRcExit $? 1 1 345export SESSION_ID=$session_id 346 347PytestCmd "test_cache_nomap.py" "test_cache_nomap_all_rows_cached" 348HandleRcExit $? 0 0 349 350DestroySession $session_id 351HandleRcExit $? 1 1 352 353GetSession 354HandleRcExit $? 1 1 355export SESSION_ID=$session_id 356 357PytestCmd "test_cache_nomap.py" "test_cache_nomap_interrupt_and_rerun" 358HandleRcExit $? 0 0 359 360DestroySession $session_id 361HandleRcExit $? 1 1 362 363for i in $(seq 1 3) 364do 365 test_name="test_cache_nomap_multiple_cache${i}" 366 GetSession 367 HandleRcExit $? 1 1 368 export SESSION_ID=$session_id 369 370 PytestCmd "test_cache_nomap.py" "${test_name}" 371 HandleRcExit $? 0 0 372 373 DestroySession $session_id 374 HandleRcExit $? 1 1 375done 376 377# Create session, run train and eval pipeline concurrently with different cache 378GetSession 379HandleRcExit $? 1 1 380export SESSION_ID=$session_id 381PytestCmd "test_cache_nomap.py" "test_cache_nomap_multiple_cache_train" & 382pids+=("$!") 383PytestCmd "test_cache_nomap.py" "test_cache_nomap_multiple_cache_eval" & 384pids+=("$!") 385 386for pid in "${pids[@]}"; do 387 wait ${pid} 388 HandleRcExit $? 0 0 389done 390 391# Running those PytestCmd in the background will not get our test_count updated. So we need to manually update it here. 392test_count=$(($test_count+1)) 393DestroySession $session_id 394HandleRcExit $? 1 1 395 396# Create session, use it to run a pipeline, and destroy the session while pipeline is running 397GetSession 398HandleRcExit $? 1 1 399export SESSION_ID=$session_id 400 401PytestCmd "test_cache_nomap.py" "test_cache_nomap_session_destroy" & 402pid=$! 403 404sleep 10 405DestroySession $session_id 406HandleRcExit $? 1 1 407wait ${pid} 408# Running those PytestCmd in the background will not get our test_count updated. So we need to manually update it here. 409test_count=$(($test_count+1)) 410 411# Stop cache server while pipeline is running 412GetSession 413HandleRcExit $? 1 1 414export SESSION_ID=$session_id 415 416PytestCmd "test_cache_nomap.py" "test_cache_nomap_server_stop" & 417pid=$! 418 419sleep 10 420StopServer 421HandleRcExit $? 1 1 422sleep 1 423wait ${pid} 424# Running those PytestCmd in the background will not get our test_count updated. So we need to manually update it here. 425test_count=$(($test_count+1)) 426 427# test cache server with --workers 1 428cmd="${CACHE_ADMIN} --start --workers 1" 429CacheAdminCmd "${cmd}" 0 430sleep 1 431HandleRcExit $? 0 0 432GetSession 433HandleRcExit $? 1 1 434export SESSION_ID=$session_id 435PytestCmd "test_cache_nomap.py" "test_cache_nomap_server_workers_1" 436HandleRcExit $? 0 0 437StopServer 438HandleRcExit $? 0 1 439 440# test cache server with --workers 100 441cmd="${CACHE_ADMIN} --start --workers 100" 442CacheAdminCmd "${cmd}" 0 443sleep 1 444HandleRcExit $? 0 0 445GetSession 446HandleRcExit $? 1 1 447export SESSION_ID=$session_id 448PytestCmd "test_cache_nomap.py" "test_cache_nomap_server_workers_100" 449HandleRcExit $? 0 0 450StopServer 451HandleRcExit $? 0 1 452 453# start cache server with a spilling path 454cmd="${CACHE_ADMIN} --start -s /tmp" 455CacheAdminCmd "${cmd}" 0 456sleep 1 457HandleRcExit $? 0 0 458 459GetSession 460HandleRcExit $? 1 1 461export SESSION_ID=$session_id 462 463# Set size parameter of mappable DatasetCache to a extra small value 464PytestCmd "test_cache_map.py" "test_cache_map_extra_small_size" 1 465HandleRcExit $? 0 0 466# Set size parameter of non-mappable DatasetCache to a extra small value 467PytestCmd "test_cache_nomap.py" "test_cache_nomap_extra_small_size" 1 468HandleRcExit $? 0 0 469 470StopServer 471HandleRcExit $? 0 1 472 473unset RUN_CACHE_TEST 474unset SESSION_ID 475 476exit ${failed_tests} 477