1#!/bin/bash 2source ./scripts/base_functions.sh 3 4# Run Export on x86 platform and create output test files: 5docker_image=mindspore_build:210301 6function Run_Export(){ 7 cd $models_path || exit 1 8 if [[ -z "${CLOUD_MODEL_ZOO}" ]]; then 9 echo "CLOUD_MODEL_ZOO is not defined - exiting export models" 10 exit 1 11 fi 12 # Export mindspore train models: 13 while read line; do 14 LFS=" " read -r -a line_array <<< ${line} 15 model_name=${line_array[0]} 16 if [[ $model_name == \#* ]]; then 17 continue 18 fi 19 echo ${model_name}'_train_export.py' >> "${export_log_file}" 20 echo 'exporting' ${model_name} 21 if [ -n "$docker_image" ]; then 22 echo 'docker run --user '"$(id -u):$(id -g)"' --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true '${docker_image}' python '${models_path}'/'${model_name}'_train_export.py' >> "${export_log_file}" 23 docker run --user "$(id -u):$(id -g)" --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true "${docker_image}" python ${models_path}'/'${model_name}_train_export.py "${epoch_num}" 24 else 25 echo 'CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} python '${models_path}'/'${model_name}'_train_export.py' >> "${export_log_file}" 26 CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} python ${models_path}'/'${model_name}_train_export.py "${epoch_num}" 27 fi 28 if [ $? = 0 ]; then 29 export_result='export mindspore '${model_name}'_train_export pass';echo ${export_result} >> ${export_result_file} 30 else 31 export_result='export mindspore '${model_name}'_train_export failed';echo ${export_result} >> ${export_result_file} 32 fi 33 done < ${models_ms_train_config} 34} 35 36# Run converter on x86 platform: 37function Run_Converter() { 38 # $1:cfgFileList 39 cd ${x86_path} || exit 1 40 tar -zxf mindspore-lite-${version}-linux-x64.tar.gz || exit 1 41 cd ${x86_path}/mindspore-lite-${version}-linux-x64/ || exit 1 42 43 cp tools/converter/converter/converter_lite ./ || exit 1 44 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./tools/converter/lib/:./tools/converter/third_party/glog/lib 45 46 rm -rf ${ms_models_path} 47 mkdir -p ${ms_models_path} 48 fail=0 49 # Convert mindspore train models: 50 while read line; do 51 LFS=" " read -r -a line_array <<< ${line} 52 local model_prefix=${line_array[0]}_train 53 parse_line convert 54 if [[ "$?" == "1" ]]; then continue; fi 55 if [[ $model_name == \#* ]]; then 56 continue 57 fi 58 if [[ "${enable_transfer}" == "1" ]]; then 59 model_prefix="${line_array[0]}_head" 60 model_name=${line_array[0]}'_head' 61 fi 62 if [[ ${check_convert} == "1" ]]; then 63 ms_file=$ms_models_path'/'$model_name'.ms' 64 if [ -f "$ms_file" ]; then 65 echo $model_name'.ms already exist, continue without convert' 66 continue 67 fi 68 fi 69 echo ${model_name} >> "${run_converter_log_file}" 70 echo './converter_lite --fmk=MINDIR --modelFile='${models_path}'/'${model_prefix}'.mindir --outputFile='${ms_models_path}'/'${model_name}' --trainModel=true' ${WEIGHT_QUANT} >> "${run_converter_log_file}" 71 ./converter_lite --fmk=MINDIR --modelFile=${models_path}/${model_prefix}.mindir --outputFile=${ms_models_path}/${model_name} --trainModel=true ${WEIGHT_QUANT} 72 if [ $? = 0 ]; then 73 converter_result='converter mindspore '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file} 74 else 75 converter_result='converter mindspore '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file} 76 fail=1 77 fi 78 # If Transfer sesstion convert backbone model 79 if [[ "${enable_transfer}" == "1" ]]; then 80 model_prefix=${line_array[0]}'_bb' 81 model_name=${line_array[0]}'_bb' 82 echo ${model_name} >> "${run_converter_log_file}" 83 echo './converter_lite --fmk=MINDIR --modelFile='${models_path}'/'${model_name}'.mindir --outputFile='${ms_models_path}'/'${model_name} ${WEIGHT_QUANT} >> "${run_converter_log_file}" 84 ./converter_lite --fmk=MINDIR --modelFile=${models_path}/${model_prefix}.mindir --outputFile=${ms_models_path}/${model_name} ${WEIGHT_QUANT} 85 if [ $? = 0 ]; then 86 converter_result='converter mindspore '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file} 87 else 88 converter_result='converter mindspore '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file} 89 fail=1 90 fi 91 fi 92 done < ${models_ms_train_config} 93 return ${fail} 94} 95 96function should_run_example() { 97 ret=0 98 while read line; do 99 LFS=" " read -r -a line_array <<< ${line} 100 model_name=${line_array[0]} 101 if [[ $model_name == \#* ]]; then 102 continue 103 fi 104 if [[ $model_name == "$1" ]]; then 105 if [[ ${line_array[1]} == "code_example" ]]; then 106 ret=1 107 fi 108 fi 109 done < ${models_ms_train_config} 110 return $ret 111} 112 113function GenerateWeightQuantConfig() { 114 echo "[common_quant_param]" > $4 115 echo "quant_type=WEIGHT_QUANT" >> $4 116 echo "bit_num=$1" >> $4 117 echo "min_quant_weight_size=$2" >> $4 118 echo "min_quant_weight_channel=$3" >> $4 119} 120 121function parse_line() { 122 i=1 123 loss_name= 124 inputShapes= 125 enable_fp16="false" 126 virtual_batch="false" 127 accuracy_limit=0.5 128 ret=0 129 WEIGHT_QUANT="" 130 enable_transfer=0 131 suffix_print="" 132 check_convert=0 133 model_name=${line_array[0]}_train 134 while [[ $i < ${#line_array[@]} ]] ; do 135 case ${line_array[i]} in 136 "weight_quant") 137 model_name="${line_array[0]}_train_quant" 138 weight_quant_config="${model_name}.cfg" 139 GenerateWeightQuantConfig 8 0 0 ${weight_quant_config} 140 WEIGHT_QUANT="--configFile=${weight_quant_config}" 141 ;; 142 "vb") 143 virtual_batch="true" 144 suffix_print="_virtual_batch" 145 check_convert=1 146 ;; 147 "loss_name") 148 i=$(($i+1)) 149 loss_name=${line_array[i]} 150 ;; 151 "noarm32") 152 if [[ $1 == "arm32" ]]; then 153 run_result=$1': '${model_name}' irrelevant'; echo ${run_result} >> ${run_benchmark_train_result_file} 154 ret=1 155 fi 156 ;; 157 "transfer") 158 enable_transfer=1 159 ;; 160 "fp16") 161 if [[ "$1" == "arm64" ]]; then 162 enable_fp16="true" 163 suffix_print="_fp16" 164 elif [[ "$1" != "convert" ]]; then 165 ret=1 166 fi 167 check_convert=1 168 ;; 169 "code_example") 170 ret=1 171 ;; 172 "inputShapes") 173 i=$(($i+1)) 174 inputShapes=${line_array[i]} 175 ;; 176 *) 177 check=`echo "${line_array[i]}" | grep -E '^\-?[0-9]*\.?[0-9]+$'` 178 if [ "${check}" != "" ] ; then 179 accuracy_limit=${line_array[i]} 180 fi 181 ;; 182 esac 183 i=$(($i+1)) 184 done 185 return $ret 186} 187 188 189# Run on x86 platform: 190function Run_x86() { 191 # $1:cfgFileList; 192 cd ${x86_path}/mindspore-lite-${version}-linux-x64 || return 1 193 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./runtime/lib:./runtime/third_party/libjpeg-turbo/lib 194 # Run mindspore converted train models: 195 local fail=0 196 while read line; do 197 198 LFS=" " read -r -a line_array <<< ${line} 199 local model_prefix=${line_array[0]} 200 local log_suffix="_train" 201 parse_line x86 202 if [[ "$?" == "1" ]]; then continue; fi 203 if [[ $model_name == \#* ]]; then 204 continue 205 fi 206 local model_file="${ms_models_path}/${model_name}.ms" 207 local bb_model_file="" 208 local export_file="${ms_models_path}/${model_name}_tod" 209 local inference_file="${ms_models_path}/${model_name}_infer" 210 if [[ "${enable_transfer}" == "1" ]]; then 211 model_file="${ms_models_path}/${model_prefix}_head.ms" 212 bb_model_file="${ms_models_path}/${model_prefix}_bb.ms" 213 suffix_print="_transfer" 214 log_suffix="_transfer" 215 export_file="${ms_models_path}/${model_prefix}_tod_head" 216 inference_file="${ms_models_path}/${model_prefix}_infer" 217 fi 218 if [ ! -z "$inference_file" ]; then 219 rm -f ${inference_file}"*" 220 fi 221 if [ ! -z "$export_file" ]; then 222 rm -f ${export_file}"*" 223 fi 224 # start running benchmark 225 local run_result 226 echo ${model_name} >> ${run_x86_log_file} 227 echo "${run_valgrind} ./tools/benchmark_train/benchmark_train \ 228 --modelFile=${model_file} \ 229 --bbModelFile=${bb_model_file} \ 230 --inDataFile=${train_io_path}/${model_prefix}_input \ 231 --expectedDataFile=${train_io_path}/${model_prefix}_output --epochs=${epoch_num} --numThreads=${threads} \ 232 --accuracyThreshold=${accuracy_limit} --inferenceFile=${inference_file} \ 233 --exportFile=${export_file} \ 234 --virtualBatch=${virtual_batch} \ 235 --inputShapes=${inputShapes} \ 236 --lossName=${loss_name} " >> ${run_x86_log_file} 237 ${run_valgrind} ./tools/benchmark_train/benchmark_train \ 238 --modelFile=${model_file} \ 239 --bbModelFile=${bb_model_file} \ 240 --inDataFile=${train_io_path}/${model_prefix}_input \ 241 --expectedDataFile=${train_io_path}/${model_prefix}_output --epochs=${epoch_num} --numThreads=${threads} \ 242 --accuracyThreshold=${accuracy_limit} --inferenceFile=${inference_file} \ 243 --exportFile=${export_file} \ 244 --virtualBatch=${virtual_batch} \ 245 --inputShapes=${inputShapes} \ 246 --lossName=${loss_name} >> "${run_x86_log_file}" 247 if [ $? = 0 ]; then 248 run_result='x86'${log_suffix}': '${model_name}''${suffix_print}' pass'; echo ${run_result} >> ${run_benchmark_train_result_file} 249 else 250 run_result='x86'${log_suffix}': '${model_name}''${suffix_print}' failed'; echo ${run_result} >> ${run_benchmark_train_result_file} 251 fail=1 252 fi 253 done < ${models_ms_train_config} 254 return ${fail} 255} 256 257# Run on arm platform: 258# Gets two parameters 259function Run_arm() { 260 # $1:platform(arm64/arm32) 261 local arm_path process_unit version_arm run_arm_log_file adb_cmd_run_file adb_push_log_file adb_cmd_file adb_cmd 262 if [ "$1" == "arm64" ]; then 263 arm_path=${arm64_path} 264 process_unit="aarch64" 265 version_arm=${version_arm64} 266 run_arm_log_file=${run_arm64_log_file} 267 adb_cmd_run_file=${adb_cmd_arm64_run_file} 268 adb_push_log_file=${adb_push_arm64_log_file} 269 adb_cmd_file=${adb_cmd_arm64_file} 270 elif [ "$1" == "arm32" ]; then 271 arm_path=${arm32_path} 272 process_unit="aarch32" 273 version_arm=${version_arm32} 274 run_arm_log_file=${run_arm32_log_file} 275 adb_cmd_run_file=${adb_cmd_arm32_run_file} 276 adb_push_log_file=${adb_push_arm32_log_file} 277 adb_cmd_file=${adb_cmd_arm32_file} 278 else 279 echo 'type ' $1 'is not supported' 280 exit 1 281 fi 282 283 # Copy the MindSpore models: 284 echo "Copy files to benchmark_train_test_arm32/64 folder, push them to the phone and run benchmark_train" 285 benchmark_train_test_path="${basepath}/benchmark_train_test_$1" 286 rm -rf ${benchmark_train_test_path} 287 mkdir -p ${benchmark_train_test_path} 288 cp -a ${ms_models_path}/*.ms ${benchmark_train_test_path} 289 290 # Unzip 291 cd ${arm_path} || exit 1 292 tar -zxf mindspore-lite-${version_arm}-android-${process_unit}.tar.gz || exit 1 293 294 # If build with minddata, copy the minddata related libs 295 cd ${benchmark_train_test_path} || exit 1 296 if [ -f ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/lib/libminddata-lite.so ]; then 297 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/third_party/libjpeg-turbo/lib/libjpeg.so* ${benchmark_train_test_path}/ || exit 1 298 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/third_party/libjpeg-turbo/lib/libturbojpeg.so* ${benchmark_train_test_path}/ || exit 1 299 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/lib/libminddata-lite.so ${benchmark_train_test_path}/libminddata-lite.so || exit 1 300 fi 301 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/third_party/hiai_ddk/lib/libhiai.so ${benchmark_train_test_path}/libhiai.so || exit 1 302 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/third_party/hiai_ddk/lib/libhiai_ir.so ${benchmark_train_test_path}/libhiai_ir.so || exit 1 303 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/third_party/hiai_ddk/lib/libhiai_ir_build.so ${benchmark_train_test_path}/libhiai_ir_build.so || exit 1 304 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/lib/libmindspore-lite.so ${benchmark_train_test_path}/libmindspore-lite.so || exit 1 305 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/runtime/lib/libmindspore-lite-train.so ${benchmark_train_test_path}/libmindspore-lite-train.so || exit 1 306 cp -a ${arm_path}/mindspore-lite-${version_arm}-android-${process_unit}/tools/benchmark_train/benchmark_train ${benchmark_train_test_path}/benchmark_train || exit 1 307 308 # adb push all needed files to the phone 309 adb -s ${device_id} push ${benchmark_train_test_path} /data/local/tmp/ > ${adb_push_log_file} 310 311 # run adb ,run session ,check the result: 312 tmp_dir="/data/local/tmp/benchmark_train_test_$1" 313 echo "cd ${tmp_dir}" > ${adb_cmd_file} 314 echo 'chmod 777 benchmark_train' >> ${adb_cmd_file} 315 adb -s ${device_id} shell < ${adb_cmd_file} 316 317 local fail=0 318 # Run mindir converted train models: 319 while read line; do 320 local line_array 321 LFS=" " read -r -a line_array <<< ${line} 322 local model_prefix=${line_array[0]} 323 local run_result="" 324 local log_suffix="_train" 325 parse_line $1 326 if [[ "$?" == "1" ]]; then continue; fi 327 if [[ $model_name == \#* ]]; then 328 continue 329 fi 330 local export_file="${tmp_dir}/${model_name}_tod" 331 local inference_file="${tmp_dir}/${model_name}_infer" 332 local model_file="${model_name}.ms" 333 local bb_model_file="" 334 if [[ "${enable_transfer}" == "1" ]]; then 335 model_file="${model_prefix}_head.ms" 336 bb_model_file="${model_prefix}_bb.ms" 337 suffix_print="_transfer" 338 export_file="${tmp_dir}/${model_prefix}_tod_head" 339 inference_file="${tmp_dir}/${model_prefix}_infer" 340 log_suffix="_transfer" 341 fi 342 # run benchmark_train test without clib data 343 echo ${model_name} >> "${run_arm_log_file}" 344 adb -s ${device_id} push ${train_io_path}/${model_prefix}_input*.bin ${train_io_path}/${model_prefix}_output*.bin ${tmp_dir} >> ${adb_push_log_file} 345 echo "cd ${tmp_dir}" > ${adb_cmd_run_file} 346 echo 'chmod 777 benchmark_train' >> ${adb_cmd_run_file} 347 if [ "$1" == arm64 ]; then 348 echo 'cp /data/local/tmp/libc++_shared.so ./' >> ${adb_cmd_run_file} 349 elif [ "$1" == arm32 ]; then 350 echo 'cp /data/local/tmp/arm32/libc++_shared.so ./' >> ${adb_cmd_run_file} 351 fi 352 adb -s ${device_id} shell < ${adb_cmd_run_file} >> ${run_arm_log_file} 353 echo "$export_file" 354 if [ ! -z "$export_file" ]; then 355 echo "rm -f ${export_file}*" >> ${run_arm_log_file} 356 echo "rm -f ${export_file}*" >> ${adb_cmd_run_file} 357 fi 358 if [ ! -z "$inference_file" ]; then 359 echo "rm -f ${inference_file}*" >> ${run_arm_log_file} 360 echo "rm -f ${inference_file}*" >> ${adb_cmd_run_file} 361 fi 362 adb -s ${device_id} shell < ${adb_cmd_run_file} >> ${run_arm_log_file} 363 adb_cmd=$(cat <<-ENDM 364 export LD_LIBRARY_PATH=./:/data/local/tmp/:${tmp_dir};./benchmark_train \ 365 --epochs=${epoch_num} \ 366 --modelFile=${model_file} \ 367 --bbModelFile=${bb_model_file} \ 368 --inDataFile=${tmp_dir}/${model_prefix}_input \ 369 --expectedDataFile=${tmp_dir}/${model_prefix}_output \ 370 --numThreads=${threads} \ 371 --accuracyThreshold=${accuracy_limit} \ 372 --enableFp16=${enable_fp16} \ 373 --inferenceFile=${inference_file} \ 374 --exportFile=${export_file} \ 375 --virtualBatch=${virtual_batch} \ 376 --inputShapes=${inputShapes} \ 377 --lossName=${loss_name} 378ENDM 379) 380 echo "${adb_cmd}" >> ${run_arm_log_file} 381 echo "${adb_cmd}" >> ${adb_cmd_run_file} 382 adb -s ${device_id} shell < ${adb_cmd_run_file} >> ${run_arm_log_file} 383 # TODO: change to arm_type 384 if [ $? = 0 ]; then 385 run_result=$1${log_suffix}': '${model_name}''${suffix_print}' pass'; echo ${run_result} >> ${run_benchmark_train_result_file} 386 else 387 run_result=$1${log_suffix}': '${model_name}''${suffix_print}' failed'; echo ${run_result} >> ${run_benchmark_train_result_file}; 388 fail=1 389 fi 390 done < ${models_ms_train_config} 391 adb -s ${device_id} shell "rm -rf ${tmp_dir}" 392 return ${fail} 393} 394 395function Run_CodeExamples() { 396 fail=0 397 target="x86" 398 tarball_path=${x86_path}/mindspore-lite-${version}-linux-x64.tar.gz 399 if [[ $backend == "arm64_train" ]]; then 400 target="arm64" 401 tarball_path=${arm64_path}/mindspore-lite-${version_arm64}-android-aarch64.tar.gz 402 export ANDROID_SERIAL=${device_id} 403 fi 404 should_run_example "train_lenet_java" 405 should_run=$? 406 407 export PATH=${x86_path}/mindspore-lite-${version}-linux-x64/tools/converter/converter/:$PATH 408 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${x86_path}/mindspore-lite-${version}-linux-x64/tools/converter/lib/:${x86_path}/mindspore-lite-${version}-linux-x64/tools/converter/third_party/glog/lib 409 410 if [[ "$should_run" == "1" && ($backend == "all" || $backend == "x86-all" || $backend == "x86_train" || $backend == "x86-java") ]]; then 411 cd ${basepath}/../../examples/train_lenet_java || exit 1 412 chmod 777 ./prepare_and_run.sh 413 ./prepare_and_run.sh -D ${datasets_path}/mnist -r ${tarball_path} -m ${models_path}/code_example.mindir >> ${run_code_examples_log_file} 414 accurate=$(tail -10 ${run_code_examples_log_file} | awk -F= 'NF==2 && /accuracy/ { sum += $2} END { print (sum > 0.80) }') 415 if [ $accurate -eq 1 ]; then 416 echo "Lenet Java Trained and reached accuracy" >> ${run_code_examples_log_file} 417 else 418 echo "train lenet java demo failure" >> ${run_code_examples_log_file} 419 fail=1 420 fi 421 cd - 422 fi 423 424 if [[ $backend == "all" || $backend == "train" || $backend == "x86_train" || $backend == "codegen_and_train" || $backend == "arm64_train" ]]; then 425 426 should_run_example "unified_api" 427 should_run=$? 428 if [[ "$should_run" == "1" ]]; then 429 cd ${basepath}/../../examples/unified_api || exit 1 430 chmod 777 ./prepare_and_run.sh 431 chmod 777 ./*/*.sh 432 ./prepare_and_run.sh -D ${datasets_path}/mnist -r ${tarball_path} -t ${target} -m ${models_path}/code_example.mindir -e 1 >> ${run_code_examples_log_file} 433 accurate=$(tail -20 ${run_code_examples_log_file} | awk 'NF==3 && /Accuracy is/ { sum += $3} END { print (sum > 1.6) }') 434 if [ $accurate -eq 1 ]; then 435 echo "Unified API Trained and reached accuracy" >> ${run_code_examples_log_file} 436 echo 'code_examples: unified_api pass' >> ${run_benchmark_train_result_file} 437 else 438 echo "Unified API demo failure" >> ${run_code_examples_log_file} 439 echo 'code_examples: unified_api failed' >> ${run_benchmark_train_result_file} 440 fail=1 441 fi 442 rm -rf package*/dataset 443 cd - 444 fi 445 446 should_run_example "train_lenet" 447 should_run=$? 448 if [[ "$should_run" == "1" ]]; then 449 cd ${basepath}/../../examples/train_lenet || exit 1 450 chmod 777 ./prepare_and_run.sh 451 chmod 777 ./*/*.sh 452 ./prepare_and_run.sh -D ${datasets_path}/mnist -r ${tarball_path} -t ${target} -m ${models_path}/code_example.mindir -e 1 >> ${run_code_examples_log_file} 453 accurate=$(tail -10 ${run_code_examples_log_file} | awk 'NF==3 && /Accuracy is/ { sum += $3} END { print (sum > 1.6) }') 454 if [ $accurate -eq 1 ]; then 455 echo "Lenet Trained and reached accuracy" >> ${run_code_examples_log_file} 456 echo 'code_examples: train_lenet pass' >> ${run_benchmark_train_result_file} 457 else 458 echo "Train Lenet demo failure" >> ${run_code_examples_log_file} 459 echo 'code_examples: train_lenet failed' >> ${run_benchmark_train_result_file} 460 fail=1 461 fi 462 rm -rf package*/dataset 463 cd - 464 fi 465 fi 466 return ${fail} 467} 468 469function Print_Result() { 470 MS_PRINT_TESTCASE_END_MSG 471 while read line; do 472 arr=("${line}") 473 printf "%-15s %-20s %-90s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]} ${arr[3]} 474 done < $1 475 MS_PRINT_TESTCASE_END_MSG 476} 477 478basepath=$(pwd) 479echo "base path is: ${basepath}" 480 481# Set models config filepath 482models_ms_train_config=${basepath}/../config/models_ms_train.cfg 483 484# Example:run_benchmark_train.sh -r /home/emir/Work/TestingEnv/release -m /home/emir/Work/TestingEnv/train_models -i /home/emir/Work/TestingEnv/train_io -d "8KE5T19620002408" 485# For running on arm64, use -t to set platform tools path (for using adb commands) 486epoch_num=1 487threads=2 488train_io_path="" 489while getopts "r:c:m:d:i:e:vt:q:D:M" opt; do 490 case ${opt} in 491 r) 492 release_path=${OPTARG} 493 echo "release_path is ${OPTARG}" 494 ;; 495 m) 496 models_path=${OPTARG} 497 echo "models_path is ${OPTARG}" 498 ;; 499 c) 500 models_ms_train_config=${OPTARG} 501 echo "models_ms_train_config ${models_ms_train_config}" 502 ;; 503 i) 504 train_io_path=${OPTARG} 505 echo "train_io_path is ${OPTARG}" 506 ;; 507 d) 508 device_id=${OPTARG} 509 echo "device_id is ${OPTARG}" 510 ;; 511 D) 512 enable_export=1 513 docker_image=${OPTARG} 514 echo "enable_export = 1, docker_image = ${OPTARG}" 515 ;; 516 e) 517 backend=${OPTARG} 518 echo "backend is ${OPTARG}" 519 ;; 520 v) 521 run_valgrind="valgrind --log-file=valgrind.log " 522 echo "Run x86 with valgrind" 523 ;; 524 q) 525 threads=${OPTARG} 526 echo "threads=${threads}" 527 ;; 528 t) 529 epoch_num=${OPTARG} 530 echo "train epoch num is ${epoch_num}" 531 ;; 532 ?) 533 echo "unknown para" 534 exit 1;; 535 esac 536done 537 538if [[ $train_io_path == "" ]]; then 539 echo "train_io path is empty" 540 train_io_path=${models_path}/input_output 541fi 542echo $train_io_path 543 544datasets_path=${models_path}/../datasets/ 545 546arm64_path=${release_path}/android_aarch64/npu 547file=$(ls ${arm64_path}/*android-aarch64.tar.gz) 548file_name="${file##*/}" 549IFS="-" read -r -a file_name_array <<< "$file_name" 550version_arm64=${file_name_array[2]} 551 552arm32_path=${release_path}/android_aarch32/npu 553file=$(ls ${arm32_path}/*android-aarch32.tar.gz) 554file_name="${file##*/}" 555IFS="-" read -r -a file_name_array <<< "$file_name" 556version_arm32=${file_name_array[2]} 557 558x86_path=${release_path}/ubuntu_x86 559file=$(ls ${x86_path}/*linux-x64.tar.gz) 560file_name="${file##*/}" 561IFS="-" read -r -a file_name_array <<< "$file_name" 562version=${file_name_array[2]} 563 564ms_models_path=${basepath}/ms_models_train 565 566logs_path=${basepath}/logs_train 567rm -rf ${logs_path} 568mkdir -p ${logs_path} 569 570# Export model if enabled 571if [[ $enable_export == 1 ]]; then 572 echo "Start Exporting models ..." 573 # Write export result to temp file 574 export_log_file=${logs_path}/export_log.txt 575 echo ' ' > ${export_log_file} 576 577 export_result_file=${logs_path}/export_result.txt 578 echo ' ' > ${export_result_file} 579 # Run export 580 Run_Export 581 Print_Result ${export_result_file} 582fi 583 584# Write converter result to temp file 585run_converter_log_file=${logs_path}/run_converter_log.txt 586echo ' ' > ${run_converter_log_file} 587 588run_converter_result_file=${logs_path}/run_converter_result.txt 589echo ' ' > ${run_converter_result_file} 590 591START=$(date +%s.%N) 592 593echo "start run converter ..." 594train_cfg_file_list=("${models_ms_train_config}") 595Run_Converter "${train_cfg_file_list[*]}" 596Run_converter_status=$? 597 598# Check converter result and return value 599if [[ ${Run_converter_status} = 0 ]];then 600 echo "Run converter success" 601 Print_Result ${run_converter_result_file} 602else 603 echo "Run converter failed" 604 cat ${run_converter_log_file} 605 Print_Result ${run_converter_result_file} 606 exit 1 607fi 608 609# Write benchmark_train result to temp file 610run_benchmark_train_result_file=${logs_path}/run_benchmark_train_result.txt 611echo ' ' > ${run_benchmark_train_result_file} 612 613# Create log files 614run_x86_log_file=${logs_path}/run_x86_log.txt 615echo 'run x86 logs: ' > ${run_x86_log_file} 616run_arm64_log_file=${logs_path}/run_arm64_log.txt 617echo 'run arm64 logs: ' > ${run_arm64_log_file} 618adb_push_arm64_log_file=${logs_path}/adb_push_arm64_log.txt 619adb_cmd_arm64_file=${logs_path}/adb_arm64_cmd.txt 620adb_cmd_arm64_run_file=${logs_path}/adb_arm64_cmd_run.txt 621 622run_arm32_log_file=${logs_path}/run_arm32_log.txt 623echo 'run arm32 logs: ' > ${run_arm32_log_file} 624adb_push_arm32_log_file=${logs_path}/adb_push_arm32_log.txt 625adb_cmd_arm32_file=${logs_path}/adb_arm32_cmd.txt 626adb_cmd_arm32_run_file=${logs_path}/adb_arm32_cmd_run.txt 627 628run_code_examples_log_file=${logs_path}/run_code_examples_log.txt 629echo 'run code examlpe logs: ' > ${run_code_examples_log_file} 630 631isFailed=0 632if [[ $backend == "all" || $backend == "train" || $backend == "x86_train" || $backend == "codegen_and_train" ]]; then 633 # Run on x86 634 echo "Start Run x86 ..." 635 Run_x86 & 636 Run_x86_PID=$! 637 sleep 1 638fi 639if [[ $backend == "all" || $backend == "train" || $backend == "x86_train" || $backend == "x86-java" || $backend == "codegen_and_train" || $backend == "arm64_train" ]]; then 640 # Run Code Examples 641 echo "Start Code Examples ..." 642 Run_CodeExamples & 643 Run_CodeExamples_PID=$! 644 sleep 1 645fi 646if [[ $backend == "all" || $backend == "train" || $backend == "arm64_train" || $backend == "codegen_and_train" ]]; then 647 # Run on arm64 648 echo "Start Run arm64 ..." 649 Run_arm arm64 & 650 Run_arm64_PID=$! 651 sleep 1 652fi 653if [[ $backend == "all" || $backend == "train" || $backend == "arm32_train" || $backend == "codegen_and_train" ]]; then 654 # Run on arm32 655 echo "Start Run arm32 ..." 656 Run_arm arm32 & 657 Run_arm32_PID=$! 658 sleep 1 659fi 660 661if [[ $backend == "all" || $backend == "train" || $backend == "x86_train" || $backend == "codegen_and_train" ]]; then 662 wait ${Run_x86_PID} 663 Run_x86_status=$? 664 if [[ ${Run_x86_status} != 0 ]];then 665 echo "Run_x86 train failed" 666 cat ${run_x86_log_file} 667 isFailed=1 668 fi 669fi 670if [[ $backend == "all" || $backend == "train" || $backend == "x86_train" || $backend == "x86-java" || $backend == "codegen_and_train" || $backend == "arm64_train" ]]; then 671 wait ${Run_CodeExamples_PID} 672 Run_CodeExamples_status=$? 673 if [[ ${Run_CodeExamples_status} != 0 ]];then 674 echo "Run CodeExamples failed" 675 cat ${run_code_examples_log_file} 676 isFailed=1 677 fi 678fi 679if [[ $backend == "all" || $backend == "train" || $backend == "arm64_train" || $backend == "codegen_and_train" ]]; then 680 wait ${Run_arm64_PID} 681 Run_arm64_status=$? 682 if [[ ${Run_arm64_status} != 0 ]];then 683 echo "Run_arm64 train failed" 684 cat ${run_arm64_log_file} 685 isFailed=1 686 fi 687fi 688if [[ $backend == "all" || $backend == "train" || $backend == "arm32_train" || $backend == "codegen_and_train" ]]; then 689 wait ${Run_arm32_PID} 690 Run_arm32_status=$? 691 if [[ ${Run_arm32_status} != 0 ]];then 692 echo "Run_arm32 train failed" 693 cat ${run_arm32_log_file} 694 isFailed=1 695 fi 696fi 697 698END=$(date +%s.%N) 699DIFF=$(echo "$END - $START" | bc) 700echo "Test ended - Results:" 701Print_Benchmark_Result $run_benchmark_train_result_file 702echo "Test run Time:" $DIFF 703exit ${isFailed} 704