1#!/bin/bash 2# Copyright 2021 Huawei Technologies Co., Ltd 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# ============================================================================ 16 17stage=0 18days=7 19iter=5 20device_id=0 21n_worker=128 22work_dir="/opt/npu/me_monitor" 23me_report_path=$work_dir/logs/ME_report_daily.xlsx 24log_dir=logs_$(date "+%m%d-%H%M") 25log_path=$work_dir/logs/$log_dir 26ms_master="https://gitee.com/mindspore/mindspore.git" 27log_data="data.json" 28ci_mode=true 29 30set -e 31set -o pipefail 32 33# parse arguments from command line 34while getopts "s:d:i:l:" args 35do 36 case $args in 37 s) 38 stage=$OPTARG 39 ;; 40 d) 41 days=$OPTARG 42 ;; 43 i) 44 iter=$OPTARG 45 ;; 46 l) 47 log_dir=$OPTARG 48 log_path=$work_dir/logs/$log_dir 49 ;; 50 ?) 51 echo "unknown argument" 52 exit 1 53 ;; 54 esac 55done 56 57source env.sh 58export DEVICE_ID=$device_id 59echo "Args: days=$days, iter=$iter, log_path=$log_path" 60cd $work_dir 61 62echo $WORKSPACE 63WORKSPACE=/home/jenkins-slave/workspace/MindSpore_Network_reid_compile_performance 64echo $WORKSPACE 65 66if [ $stage -le 1 ]; then 67 echo "" 68 echo "===========Stage 1: Fetching latest mindspore from master===========" 69 if [ -d mindspore ]; then 70 rm -rf mindspore 71 fi 72 git clone $ms_master 73fi 74 75if [ $stage -le 2 ]; then 76 echo "" 77 echo "===========Stage 2: Building mindspore===========" 78 cd $work_dir/mindspore 79 bash build.sh -e ascend -j $n_worker -p on 80fi 81 82if [ $stage -le 3 ]; then 83 echo "" 84 echo "===========Stage 3: Compiling networks===========" 85 cd $work_dir 86 mkdir -p $log_path 87 88 # Compiling ReID-8 89 # split resource-consuming task from others 90 for count in $(seq 1 $iter); do 91 echo "[INFO] Compiling ReID-8p, iteration $count" 92 if [ -d reid$count ]; then 93 rm -rf reid$count 94 fi 95 mkdir reid$count 96 cd reid$count 97 bash $work_dir/faceReidToMe/dist_env/env_26/dist_env_26.sh 98 for num in {0..7}; do 99 cp device_$num/test_reid_stage123_1024node_graphdata_dynamiclossscale_log$num.log $log_path/reid_${count}_${num}.log 100 done 101 cd $work_dir 102 mv reid$count $log_path 103 done 104 105 # Compiling BERT 106 cd $work_dir 107 for count in $(seq 1 $iter); do 108 echo "[INFO] Compiling BERT, iteration $count" 109 pytest -s mindspore/tests/perf_test/bert/test_bert_train.py::test_bert_train | tee $log_path/bert$count.log 110 done 111 112 # Compiling ResNet50 113 for count in $(seq 1 $iter); do 114 echo "[INFO] Compiling ResNet50, iteration $count" 115 pytest -s mindspore/tests/perf_test/test_resnet_train.py::test_train_step | tee $log_path/resnet$count.log 116 done 117 118 # Compiling GPT 119 for count in $(seq 1 $iter); do 120 echo "[INFO] Compiling GPT, iteration $count" 121 cd gpt 122 bash scripts/run_standalone_train.sh 0 1 $work_dir/gpt_data | tee $log_path/gpt$count.log 123 done 124fi 125 126if [ $stage -le 4 ]; then 127 echo "" 128 echo "===========Stage 4: Processing log files===========" 129 cd $work_dir 130 python process_data.py $me_report_path $log_path $iter $log_path/$log_data 131fi 132 133if [ $stage -le 5 ]; then 134 echo "" 135 echo "===========Stage 5: Generating reports===========" 136 if [ ! -d $log_path/reports ]; then 137 mkdir $log_path/reports 138 fi 139 python generate_report.py $log_path $log_path/$log_data $me_report_path $days 140 141 if [ $ci_mode ]; then 142 echo "copying file to artifacts" 143 mkdir -p ${WORKSPACE}/archive 144 cp $log_path/reports/* ${WORKSPACE}/archive 145 fi 146fi 147