1#!/bin/bash 2# Copyright 2022 Huawei Technologies Co., Ltd 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# init parameter 17status=0 18REPOSITORY_NAME=mindspore 19PIPELINE_TYPE="gate" 20CURR_DIR=$(dirname "${BASH_SOURCE-$0}") 21CURR_DIR=$( 22 cd -P "${CURR_DIR}" || exit 23 pwd -P 24) 25REPO_HOME=$( 26 cd -P "${CURR_DIR}/../../../" || exit 27 pwd -P 28) 29WORKSPACE=$( 30 cd -P "${CURR_DIR}/../../../../" || exit 31 pwd -P 32) 33 34# get os name 35uname_s=$(uname -s) 36os_name=${uname_s:0:5} 37if [ "$os_name" == "Darwi" ]; then # Darwin 38 echo "Mac OS X" 39elif [ "$os_name" == "Linux" ]; then # Linux 40 echo "GNU/Linux" 41elif [ "$os_name" == "MINGW" ]; then # MINGW, windows, git-bash 42 echo "Windows, git-bash" 43else 44 echo "unknown os" 45fi 46 47STAGE_FILES=$(git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD) 48if [[ "$STAGE_FILES" == "" && $PIPELINE_TYPE == "gate" ]]; then 49 echo "No file need to push." 50 exit "${status}" 51fi 52 53# mkdir check_path 54check_path=${WORKSPACE}/.${REPOSITORY_NAME}_code_check 55rm -rf "${check_path}" 56mkdir -p "${check_path}" 57 58# mkdir CODE_PATH 59CODE_PATH="${check_path}/code" 60mkdir -p "${CODE_PATH}" 61 62# get gate pr filelist 63if [ "${PIPELINE_TYPE}" = "gate" ]; then 64 cd "${REPO_HOME}" || return 65 git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD >"${CODE_PATH}/pr_filelist.txt" 66 cat "${CODE_PATH}/pr_filelist.txt" 67 echo "Filelist number: $(cat <"${CODE_PATH}"/pr_filelist.txt | wc -l)" 68 69 mkdir "${CODE_PATH}/${REPOSITORY_NAME}" 70 while read -r line; do 71 file_dirname=$(dirname "${line}") 72 mkdir -p "${CODE_PATH}/${REPOSITORY_NAME}/${file_dirname}" 73 cp -a "${REPO_HOME}/${line}" "${CODE_PATH}/${REPOSITORY_NAME}/${file_dirname}" >/dev/null 2>&1 74 done <"${CODE_PATH}/pr_filelist.txt" 75 76 # Reserved directory(.jenkins) for code check 77 if [ -d "${REPO_HOME}/.jenkins" ]; then 78 cp -a "${REPO_HOME}/.jenkins" "${CODE_PATH}"/${REPOSITORY_NAME}/.jenkins >/dev/null 2>&1 79 fi 80fi 81 82# Common exclude folder for code check 83if [ "${REPOSITORY_NAME}" = "mindspore" ]; then 84 COMMON_EXCLUDE_FOLDER="third_party,graphengine,akg,mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils,mindspore/lite/micro/example,tests/models,mindspore/lite/examples/quick_start_micro" 85elif [ "${REPOSITORY_NAME}" = "akg" ]; then 86 COMMON_EXCLUDE_FOLDER="third_party,tests" 87else 88 COMMON_EXCLUDE_FOLDER="third_party" 89fi 90 91# Get CODE 92function GET_CODE() { 93 workspace=${1} 94 if [ "${PIPELINE_TYPE}" = "gate" ]; then 95 if [ "$os_name" == "Darwi" ]; then 96 cp -a "${CODE_PATH}/${REPOSITORY_NAME}/" "${workspace}/${REPOSITORY_NAME}/" >/dev/null 2>&1 97 else 98 cp -a "${CODE_PATH}/${REPOSITORY_NAME}/" "${workspace}"/ >/dev/null 2>&1 99 fi 100 else 101 cp -a "${REPO_HOME}/" "${workspace}/${REPOSITORY_NAME}" >/dev/null 2>&1 102 fi 103} 104 105# Exclude folder 106function MS_EXCLUDE_FOLDER() { 107 local src_path=${1} 108 local exclude_folder="${2}" 109 110 if [ -n "${src_path}" ] && [ -n "$(echo "${src_path}" | awk -F'/' '{print $3}')" ]; then 111 for folder in ${exclude_folder//,/ }; do 112 rm -rf "${src_path:?}/${folder:?}" 113 done 114 else 115 return 116 fi 117} 118 119# judge succeed or failed 120function DP_ASSERT_EQUAL() { 121 local actual_value=${1} 122 local expect_value=${2} 123 local assert_msg=${3} 124 if [ "${actual_value}" != "${expect_value}" ]; then 125 echo "${assert_msg} failed, ${assert_msg} error number is ${actual_value}." 126 status=$((status + actual_value)) 127 else 128 echo "${assert_msg} succeed." 129 fi 130} 131 132# Assert A is not equal to B 133# True: not equal 134# False: equal 135function DP_ASSERT_NOT_EQUAL() { 136 local actual_value=${1} 137 local expect_value=${2} 138 local assert_msg=${3} 139 if [ "${actual_value}" = "${expect_value}" ]; then 140 echo "${assert_msg} is failed, ${assert_msg} error number is ${actual_value}." 141 status=$((status + 1)) 142 else 143 echo "${assert_msg} succeed." 144 fi 145} 146 147# Print N character 148function PRINT_N_CHAR() { 149 local char=${1} 150 local number=${2} 151 local str 152 str=$(printf "%-${number}s" "") 153 echo "${str// /${char}}" 154} 155 156# 进行clang-format检查 157function clang_format_check() { 158 if [ -f "${REPO_HOME}/scripts/check_clang_format.sh" ]; then 159 echo "clang-format scan start" 160 which clang-format || (echo "[INFO] Please install 'clang-format' tool first" && return 1) 161 if [ $? -eq "1" ]; then 162 return 163 fi 164 165 # create clang-format workspace 166 local workspace=${CODE_PATH}/clang_format 167 rm -rf "${workspace}" 168 mkdir -p "${workspace}" 169 170 # get code 171 cp -a "${REPO_HOME}/" "${workspace}/${REPOSITORY_NAME}" >/dev/null 2>&1 172 173 # Reset exclude file 174 local LOCAL_EXCLUDE_FOLDER=${COMMON_EXCLUDE_FOLDER} 175 if [ "${REPOSITORY_NAME}" = "mindspore" ]; then 176 local DVPP_PATH="mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils" 177 LOCAL_EXCLUDE_FOLDER=$(echo ${LOCAL_EXCLUDE_FOLDER} | sed "s#${DVPP_PATH},##g" | sed "s#,${DVPP_PATH}##g") 178 fi 179 180 # Exclude folder 181 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 182 183 # Check clang-format 184 echo "clang-format scanning" 185 cd "${workspace}/${REPOSITORY_NAME}" || return 186 if [ "${PIPELINE_TYPE}" = "gate" ]; then 187 bash -x scripts/check_clang_format.sh -l 188 else 189 bash -x scripts/check_clang_format.sh -a 190 fi 191 DP_ASSERT_EQUAL "$?" "0" "Clang-format scanning" 192 else 193 echo "${REPO_HOME}/scripts/check_clang_format.sh is not exist." 194 fi 195} 196 197# 进行cmakelint检查 198function cmakelint_check() { 199 echo "cmakelint scan start" 200 which cmakelint || (echo "[INFO] Please install 'cmakelint' tool first" && return 1) 201 if [ $? -eq "1" ]; then 202 return 203 fi 204 205 # create cmakelint workspace 206 local workspace=${CODE_PATH}/cmakelint 207 rm -rf "${workspace}" 208 mkdir -p "${workspace}" 209 210 # get code 211 GET_CODE "${workspace}" 212 213 # Exclude folder 214 local LOCAL_EXCLUDE_FOLDER=$COMMON_EXCLUDE_FOLDER 215 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 216 217 # Set rules 218 local CMAKELINT_FILTER="+convention/filename,+linelength,+package/consistency,+readability/logic,-readability/mixedcase,-readability/wonkycase,-syntax,+whitespace/eol,+whitespace/extra,+whitespace/indent,+whitespace/mismatch,+whitespace/newline,+whitespace/tabs" 219 220 # create cmakelint_check path 221 local cmakelint_path=${check_path}/cmakelint 222 mkdir "${cmakelint_path}" 223 : >"${cmakelint_path}"/cmakelint.log 224 225 # Run cmakelint 226 echo "cmakelint scanning" 227 cd "${workspace}" || return 228 find ${REPOSITORY_NAME} -name '*.cmake' -o -name 'CMakeLists.txt' -o -name 'cmakelists.txt' | xargs -r cmakelint --filter=${CMAKELINT_FILTER} --spaces=2 --linelength=120 --quiet >"${cmakelint_path}"/cmakelint.log 229 230 if [ "$os_name" == "MINGW" ]; then 231 # change \ to / in "${cmakelint_path}"/cmakelint.log 232 sed -i 's#\\#\/#g' "${cmakelint_path}"/cmakelint.log 233 fi 234 235 error_number=$(cat <"${cmakelint_path}"/cmakelint.log | grep -c "^${REPOSITORY_NAME}/") 236 if [ "$error_number" != 0 ]; then 237 # Print content 238 echo "Problem items: " 239 cat <"${cmakelint_path}"/cmakelint.log | grep "^${REPOSITORY_NAME}/" 240 fi 241 # Return result 242 DP_ASSERT_EQUAL "${error_number}" "0" "Cmakelint scanning" 243} 244 245# 进行codespell检查 246function codespell_check() { 247 echo "codespell scan start" 248 which codespell || (echo "[INFO] Please install 'codespell' tool first" && return 1) 249 if [ $? -eq "1" ]; then 250 return 251 fi 252 253 # create codespell workspace 254 local workspace=${CODE_PATH}/codespell 255 rm -rf "${workspace}" 256 mkdir -p "${workspace}" 257 258 # get code 259 GET_CODE "${workspace}" 260 261 # Reset exclude file 262 local LOCAL_EXCLUDE_FOLDER=$COMMON_EXCLUDE_FOLDER 263 if [ "${REPOSITORY_NAME}" = "mindspore" ]; then 264 LOCAL_EXCLUDE_FOLDER="${LOCAL_EXCLUDE_FOLDER},tests/ut/data,mindspore/python/mindspore/profiler/common/validator/validate.py,mindspore/python/mindspore/ops/_op_impl/_custom_op/fake_quant_perlayer.py" 265 fi 266 267 # Exclude folder 268 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 269 270 # Check rule file 271 if [ -f "${workspace}/${REPOSITORY_NAME}/.jenkins/rules/codespell/codespell.allow" ]; then 272 local RULE_FILE=${workspace}/${REPOSITORY_NAME}/.jenkins/rules/codespell/codespell.allow 273 else 274 echo "${REPOSITORY_NAME}/.jenkins/rules/codespell/codespell.allow not exist" 275 return 276 fi 277 278 # mkdir codespell_path 279 local codespell_path=${check_path}/codespell 280 mkdir "${codespell_path}" 281 : >"${codespell_path}"/codespell.log 282 283 # Run codespell 284 echo "codespell scanning" 285 cd "${workspace}" || return 286 codespell -q 7 -S '.git,third_party' -I "${RULE_FILE}" ${REPOSITORY_NAME} | grep -E -v ": [0-9a-zA-Z]{1,4} ==> |: [a-zA-Z][a-z]{1,}[TXYZ] ==> |: [a-z][A-Z][a-z]{1,} ==> " >"${codespell_path}"/codespell.log 287 288 if [ "$os_name" == "MINGW" ]; then 289 # change \ to / in "${codespell_path}"/codespell.log 290 sed -i 's#\\#\/#g' "${codespell_path}"/codespell.log 291 fi 292 293 # filter codespell 294 cat <"${RULE_FILE}" | grep -v "^[[:space:]]*$" | grep -v "^#" | tr -d '\r' >"${codespell_path}"/codespell_filter.txt 295 while read -r line; do 296 if [ "$os_name" == "Darwi" ]; then 297 sed -i "" "/: ${line} ==> /d" "${codespell_path}"/codespell.log 298 else 299 sed -i "/: ${line} ==> /d" "${codespell_path}"/codespell.log 300 fi 301 done <"${codespell_path}"/codespell_filter.txt 302 303 error_number=$(cat <"${codespell_path}"/codespell.log | grep -c "^${REPOSITORY_NAME}/") 304 if [ "$error_number" != 0 ]; then 305 # Print content 306 echo "Problem items: " 307 cat <"${codespell_path}"/codespell.log | grep "^${REPOSITORY_NAME}/" 308 fi 309 # Return result 310 DP_ASSERT_EQUAL "${error_number}" "0" "Codespell scanning" 311} 312 313# 进行cpplint检查 314function cpplint_check() { 315 echo "cpplint scan start" 316 which cpplint || (echo "[INFO] Please install 'cpplint' tool first" && return 1) 317 if [ $? -eq "1" ]; then 318 return 319 fi 320 321 # create cpplint workspace 322 local workspace=${CODE_PATH}/cpplint 323 rm -rf "${workspace}" 324 mkdir -p "${workspace}" 325 326 # get code 327 GET_CODE "${workspace}" 328 329 # Reset exclude file 330 local LOCAL_EXCLUDE_FOLDER=$COMMON_EXCLUDE_FOLDER 331 if [ "${REPOSITORY_NAME}" = "mindspore" ]; then 332 LOCAL_EXCLUDE_FOLDER="${LOCAL_EXCLUDE_FOLDER},tests" 333 elif [ "${REPOSITORY_NAME}" = "models" ]; then 334 LOCAL_EXCLUDE_FOLDER="${LOCAL_EXCLUDE_FOLDER},official/audio/lpcnet/third_party/src" 335 fi 336 337 # Exclude folder 338 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 339 340 # Run cpplint 341 echo "cpplint scanning" 342 local cpplint_path=${check_path}/cpplint 343 mkdir "${cpplint_path}" 344 : >"${cpplint_path}"/cpplint.log 345 cd "${workspace}" || return 346 cpplint --root=src --extensions=cxx,cu,hh,cpp,hxx,cuh,h++,cc,c,hpp,c++,h,tpp,txx,cl --filter=-build/header_guard,-build/c++11 --quiet --repository="${workspace}"/${REPOSITORY_NAME} --linelength=120 --recursive ${REPOSITORY_NAME} >"${cpplint_path}"/cpplint.log 2>&1 347 348 if [ "$os_name" == "MINGW" ]; then 349 # change \ to / in "${cpplint_path}"/cpplint.log 350 sed -i 's#\\#\/#g' "${cpplint_path}"/cpplint.log 351 fi 352 353 # Filter 354 rm -f "${cpplint_path}"/cpplint_org.log 355 cp -a "${cpplint_path}"/cpplint.log "${cpplint_path}"/cpplint_org.log >/dev/null 2>&1 356 if [ -f "${workspace}/${REPOSITORY_NAME}/.jenkins/check/config/filter_cpplint.txt" ]; then 357 local FILTER_FILE=${workspace}/${REPOSITORY_NAME}/.jenkins/check/config/filter_cpplint.txt 358 fi 359 cat <"${FILTER_FILE}" | grep -v "^[[:space:]]*$" | grep -v "^#" | tr -d '\r' >"${cpplint_path}"/cpplint_filter.txt 360 while read -r line; do 361 local key1 362 key1=$(echo "${line}" | awk -F'"' '{print $2}') 363 local key2 364 key2=$(echo "${line}" | awk -F'"' '{print $4}') 365 cat <"${cpplint_path}"/cpplint.log | grep -n "^${key1}" | grep "${key2}" | awk -F':' '{print $1}' >"${cpplint_path}"/cpplint_line.txt 366 for line_number in $(tac "${cpplint_path}"/cpplint_line.txt); do 367 if [ "$os_name" == "Darwi" ]; then 368 sed -i "" "${line_number}d" "${cpplint_path}"/cpplint.log 369 else 370 sed -i "${line_number}d" "${cpplint_path}"/cpplint.log 371 fi 372 done 373 done <"${cpplint_path}"/cpplint_filter.txt 374 375 # Check result 376 if [ -s "${cpplint_path}"/cpplint_org.log ] && [ ! -s "${cpplint_path}"/cpplint.log ]; then 377 echo "[ERROR] Filter cpplint failed." 378 fi 379 380 error_number=$(cat <"${cpplint_path}"/cpplint.log | grep -c "^${REPOSITORY_NAME}/") 381 if [ "$error_number" != 0 ]; then 382 # Print content 383 echo "Problem items: " 384 cat <"${cpplint_path}"/cpplint.log | grep "^${REPOSITORY_NAME}/" 385 fi 386 # Return result 387 DP_ASSERT_EQUAL "${error_number}" "0" "Cpplint scanning" 388} 389 390# 进行lizard检查 391function lizard_check() { 392 echo "lizard scan start" 393 which lizard || (echo "[INFO] Please install 'lizard' tool first" && return 1) 394 if [ $? -eq "1" ]; then 395 return 396 fi 397 398 # create lizard workspace 399 local workspace=${CODE_PATH}/lizard 400 rm -rf "${workspace}" 401 mkdir -p "${workspace}" 402 403 # get code 404 GET_CODE "${workspace}" 405 406 # Exclude folder 407 local LOCAL_EXCLUDE_FOLDER=${COMMON_EXCLUDE_FOLDER} 408 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 409 410 # Set additional option 411 local ADDITIONAL_LIZARD_OPTION="" 412 if [ -f "${workspace}/${REPOSITORY_NAME}/.jenkins/check/config/whitelizard.txt" ]; then 413 ADDITIONAL_LIZARD_OPTION="-W ${workspace}/${REPOSITORY_NAME}/.jenkins/check/config/whitelizard.txt" 414 fi 415 416 # mkdir lizard_path 417 local lizard_path=${check_path}/lizard 418 mkdir "${lizard_path}" 419 : >"${lizard_path}"/lizard.log 420 421 # Run lizard 422 echo "lizard scanning" 423 cd "${workspace}" || return 424 local THRESHOLD_LIZARD_CCN=19 425 local THRESHOLD_LIZARD_LENGTH=100 426 lizard -l cpp -l python -l java -C ${THRESHOLD_LIZARD_CCN} -L ${THRESHOLD_LIZARD_LENGTH} -x "*/tests/*" -x "*/test/*" -x "*/third_party/*" "${ADDITIONAL_LIZARD_OPTION}" -w ${REPOSITORY_NAME} >"${lizard_path}"/lizard.log 427 428 if [ "$os_name" == "MINGW" ]; then 429 # change \ to / in "${lizard_path}"/lizard.log 430 sed -i 's#\\#\/#g' "${lizard_path}"/lizard.log 431 fi 432 433 # Get result of cyclomatic complexity 434 ( 435 while read -r line; do 436 filename=$(echo "${line}" | awk -F': ' '{print $1}') 437 method_name=$(echo "${line}" | awk -F': ' '{print $3}' | awk '{print $1}') 438 method_ccn=$(echo "${line}" | awk -F'NLOC, ' '{print $2}' | awk -F' CCN' '{print $1}') 439 if [[ ${method_ccn} -gt ${THRESHOLD_LIZARD_CCN} ]]; then 440 printf "%-115s %-75s %-7s\n" "${filename}" "${method_name}" "${method_ccn}" 441 fi 442 done <"${lizard_path}"/lizard.log 443 ) >"${lizard_path}"/lizard_ccn.log 444 445 # Get result of large function 446 ( 447 while read -r line; do 448 filename=$(echo "${line}" | awk -F': ' '{print $1}') 449 method_name=$(echo "${line}" | awk -F': ' '{print $3}' | awk '{print $1}') 450 method_nloc=$(echo "${line}" | awk -F'has ' '{print $2}' | awk -F' NLOC' '{print $1}') 451 if [[ ${method_nloc} -gt ${THRESHOLD_LIZARD_LENGTH} ]]; then 452 printf "%-115s %-75s %-7s\n" "${filename}" "${method_name}" "${method_nloc}" 453 fi 454 done <"${lizard_path}"/lizard.log 455 ) >"${lizard_path}"/lizard_nloc.log 456 457 error_number_ccn=$(cat <"${lizard_path}"/lizard_ccn.log | grep -c "^${REPOSITORY_NAME}/") 458 if [ "$error_number_ccn" != 0 ]; then 459 # Print content of cyclomatic complexity 460 echo "[Lizard] Cyclomatic complexity error number: ${error_number_ccn}, threshold(CCN) > ${THRESHOLD_LIZARD_CCN} problem items: " 461 if [ -s "${lizard_path}/lizard_ccn.log" ]; then 462 PRINT_N_CHAR "-" "195" 463 printf "%-115s %-75s %-7s\n" "FilePath" "Function" "CCN" 464 printf "%-115s %-75s %-7s\n" "--------" "--------" "---" 465 cat "${lizard_path}"/lizard_ccn.log 466 PRINT_N_CHAR "-" "195" 467 fi 468 fi 469 470 error_number_nloc=$(cat <"${lizard_path}"/lizard_nloc.log | grep -c "^${REPOSITORY_NAME}/") 471 if [ "$error_number_nloc" != 0 ]; then 472 # Print content of large function 473 echo "[Lizard] Function length error number: ${error_number_nloc}, threshold(NLOC) > ${THRESHOLD_LIZARD_LENGTH} problem items: " 474 if [ -s "${lizard_path}/lizard_nloc.log" ]; then 475 PRINT_N_CHAR "-" "196" 476 printf "%-115s %-75s %-7s\n" "FilePath" "Function" "NLOC" 477 printf "%-115s %-75s %-7s\n" "--------" "--------" "----" 478 cat "${lizard_path}"/lizard_nloc.log 479 PRINT_N_CHAR "-" "196" 480 fi 481 fi 482 # Return result 483 DP_ASSERT_EQUAL "$((error_number_ccn + error_number_nloc))" "0" "Lizard scanning" 484} 485 486# 进行markdownlint检查 487function markdownlint_check() { 488 echo "markdownlint scan start" 489 which mdl || (echo "[INFO] Please install 'markdownlint' tool first" && return 1) 490 if [ $? -eq "1" ]; then 491 return 492 fi 493 494 # create markdownlint workspace 495 local workspace=${CODE_PATH}/markdownlint 496 rm -rf "${workspace}" 497 mkdir -p "${workspace}" 498 499 # get code 500 GET_CODE "${workspace}" 501 502 # Exclude folder 503 local LOCAL_EXCLUDE_FOLDER=$COMMON_EXCLUDE_FOLDER 504 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 505 506 # Get rules 507 local RULES_FILE_NAME="markdownlint_mindspore.rb" 508 if [ -f "${workspace}/${REPOSITORY_NAME}/.jenkins/rules/markdownlint/${RULES_FILE_NAME}" ]; then 509 local RULE_FILE=${workspace}/${REPOSITORY_NAME}/.jenkins/rules/markdownlint/${RULES_FILE_NAME} 510 fi 511 512 # Run markdownlint 513 echo "markdownlint scanning" 514 cd "${workspace}" || return 515 mdl -s "${RULE_FILE}" ${REPOSITORY_NAME} 516 DP_ASSERT_EQUAL "$?" "0" "Markdownlint scanning" 517} 518 519# 进行pylint检查 520function pylint_check() { 521 echo "pylint scan start" 522 which pylint || (echo "[INFO] Please install 'pylint' tool first" && return 1) 523 if [ $? -eq "1" ]; then 524 return 525 fi 526 527 # create pylint workspace 528 local workspace=${CODE_PATH}/pylint 529 rm -rf "${workspace}" 530 mkdir -p "${workspace}" 531 532 # get code 533 GET_CODE "${workspace}" 534 535 # Reset exclude file 536 local LOCAL_EXCLUDE_FOLDER=$COMMON_EXCLUDE_FOLDER 537 if [ "${PIPELINE_TYPE}" = "daily" ] || [ "${PIPELINE_TYPE}" = "version" ]; then 538 LOCAL_EXCLUDE_FOLDER="${LOCAL_EXCLUDE_FOLDER},tests" 539 fi 540 LOCAL_EXCLUDE_FOLDER="${LOCAL_EXCLUDE_FOLDER},mindspore/profiler/common/proto_files" 541 542 # Exclude folder 543 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 544 545 # Check rule file 546 if [ -f "${workspace}/${REPOSITORY_NAME}/.jenkins/rules/pylint/pylintrc" ]; then 547 local RULE_FILE=${workspace}/${REPOSITORY_NAME}/.jenkins/rules/pylint/pylintrc 548 fi 549 550 # create pylint path 551 local pylint_path=${check_path}/pylint 552 mkdir -p "${pylint_path}" 553 554 # Get scan filename 555 if [ "${PIPELINE_TYPE}" = "gate" ]; then 556 # Get pylint pr filelist(*.py) 557 cat <"${CODE_PATH}"/pr_filelist.txt | grep '\.py$' >"${pylint_path}"/scan_filename.txt 558 else 559 # Get scan file number 560 cd "${workspace}" || return 561 (find ${REPOSITORY_NAME} -iname "*.py") >"${pylint_path}"/scan_filename.txt 562 if [ "$os_name" == "Darwi" ]; then 563 sed -i "" "s#^${REPOSITORY_NAME}/##g" "${pylint_path}"/scan_filename.txt 564 else 565 sed -i "s#^${REPOSITORY_NAME}/##g" "${pylint_path}"/scan_filename.txt 566 fi 567 fi 568 569 # run pylint 570 echo "pylint scanning" 571 cd "${workspace}" || return 572 : >"${pylint_path}"/pylint.log 573 while read -r line; do 574 if [ -f "${workspace}/${REPOSITORY_NAME}/${line}" ]; then 575 pylint --rcfile="${RULE_FILE}" -j 4 --output-format=parseable "${REPOSITORY_NAME}/${line}" --max-line-length=120 >>"${pylint_path}"/pylint.log 576 fi 577 done <"${pylint_path}"/scan_filename.txt 578 579 if [ "$os_name" == "MINGW" ]; then 580 # change \ to / in "${pylint_path}"/lizard.log 581 sed -i 's#\\#\/#g' "${pylint_path}"/pylint.log 582 fi 583 584 # filter pylint 585 rm -f "${pylint_path}"/pylint_org.log 586 cp -a "${pylint_path}"/pylint.log "${pylint_path}"/pylint_org.log >/dev/null 2>&1 587 if [ -f "${workspace}/${REPOSITORY_NAME}/.jenkins/check/config/filter_pylint.txt" ]; then 588 local FILTER_FILE=${workspace}/${REPOSITORY_NAME}/.jenkins/check/config/filter_pylint.txt 589 fi 590 cat <"${FILTER_FILE}" | grep -v "^[[:space:]]*$" | grep -v "^#" | tr -d '\r' >"${pylint_path}"/pylint_filter.txt 591 while read -r line; do 592 local key1 593 key1=$(echo "${line}" | awk -F'"' '{print $2}') 594 local key2 595 key2=$(echo "${line}" | awk -F'"' '{print $4}') 596 cat <"${pylint_path}"/pylint.log | grep -n "^${key1}" | grep "${key2}" | awk -F':' '{print $1}' >"${pylint_path}"/pylint_line.txt 597 for line_number in $(tac "${pylint_path}"/pylint_line.txt); do 598 if [ "$os_name" == "Darwi" ]; then 599 sed -i "" "${line_number}d" "${pylint_path}"/pylint.log 600 else 601 sed -i "${line_number}d" "${pylint_path}"/pylint.log 602 fi 603 done 604 done <"${pylint_path}"/pylint_filter.txt 605 606 # Check result 607 if [ -s "${pylint_path}"/pylint_org.log ] && [ ! -s "${pylint_path}"/pylint.log ]; then 608 echo "[ERROR] Filter pylint failed." 609 return 610 fi 611 612 error_number=$(cat <"${pylint_path}"/pylint.log | grep -c "^${REPOSITORY_NAME}/") 613 if [ "$error_number" != 0 ]; then 614 # Print content 615 echo "Problem items: " 616 cat <"${pylint_path}"/pylint.log | grep "^${REPOSITORY_NAME}/" 617 fi 618 # Return result 619 DP_ASSERT_EQUAL "${error_number}" "0" "Pylint scanning" 620} 621 622# 进行shellcheck检查 623function shellcheck_check() { 624 echo "shellcheck scan start" 625 which shellcheck || (echo "[INFO] Please install 'shellcheck' tool first" && return 1) 626 if [ $? -eq "1" ]; then 627 return 628 fi 629 630 # create shellcheck workspace 631 local workspace=${CODE_PATH}/shellcheck 632 rm -rf "${workspace}" 633 mkdir -p "${workspace}" 634 635 # get code 636 GET_CODE "${workspace}" 637 638 # Exclude folder 639 local LOCAL_EXCLUDE_FOLDER=$COMMON_EXCLUDE_FOLDER 640 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 641 642 # check shellcheck path 643 local shellcheck_path=${check_path}/shellcheck 644 mkdir -p "${shellcheck_path}" 645 : >"${shellcheck_path}"/shellcheck_result.log 646 647 # Filter 648 # Reference: https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/.shellcheckrc 649 # SC2086: Double quote to prevent globbing and word splitting. 650 # SC1090: Can't follow non-constant source. Use a directive to specify location. 651 # SC1091: Not following: (error message here) 652 # SC2155: Declare and assign separately to avoid masking return values. 653 # SC2164: Use cd ... || exit in case cd fails. 654 # SC1003: Want to escape a single quote? echo 'This is how it'\''s done'. 655 local SHELLCHECK_EXCLUDE_TYPES="SC2086,SC1090,SC1091,SC2155,SC2164,SC1003" 656 657 # Run shellcheck (warning) 658 echo "shellcheck scanning" 659 local error_number 660 error_number=$(find "${workspace}"/${REPOSITORY_NAME} -name '*.sh' -type f -print0 | xargs -0 -r shellcheck --severity=warning --exclude=${SHELLCHECK_EXCLUDE_TYPES} --format=gcc | wc -l) 661 if [ "${error_number}" -ne "0" ]; then 662 find "${workspace}"/${REPOSITORY_NAME} -name '*.sh' -type f -print0 | xargs -0 -r shellcheck --severity=warning --exclude=${SHELLCHECK_EXCLUDE_TYPES} --format=tty >"${shellcheck_path}"/shellcheck_result.log 663 # Print content 664 echo "Problem items: " 665 cat "${shellcheck_path}/shellcheck_result.log" 666 fi 667 DP_ASSERT_EQUAL "${error_number}" "0" "Shellcheck scanning" 668} 669 670# 进行tab检查 671function tab_check() { 672 echo "tab scan start" 673 674 # create tab workspace 675 local workspace=${CODE_PATH}/tab 676 rm -rf "${workspace}" 677 mkdir -p "${workspace}" 678 679 # get code 680 if [ "${PIPELINE_TYPE}" = "gate" ]; then 681 if [ "$os_name" == "Darwi" ]; then 682 cp -a "${CODE_PATH}/${REPOSITORY_NAME}/" "${workspace}/${REPOSITORY_NAME}/" >/dev/null 2>&1 683 else 684 cp -a "${CODE_PATH}/${REPOSITORY_NAME}/" "${workspace}"/ >/dev/null 2>&1 685 fi 686 # Reserved directory(.git) for command(git grep) 687 cp -a "${REPO_HOME}/.git" "${workspace}/${REPOSITORY_NAME}/" >/dev/null 2>&1 688 else 689 cp -a "${REPO_HOME}/" "${workspace}/${REPOSITORY_NAME}" >/dev/null 2>&1 690 fi 691 692 # Reset exclude file 693 local LOCAL_EXCLUDE_FOLDER=$COMMON_EXCLUDE_FOLDER 694 if [ "${REPOSITORY_NAME}" = "mindspore" ]; then 695 LOCAL_EXCLUDE_FOLDER="${LOCAL_EXCLUDE_FOLDER},tests/ut/data,tests/ut/cpp/dataset/c_api_text_test.cc,mindspore/lite/examples/quick_start_ios/mindspore-lite.xcodeproj,mindspore/lite/examples/quick_start_ios/mindspore-lite/Base.Iproj,mindspore/lite/examples/quick_start_ios/mindspore-lite/Assets.xcassets,mindspore/lite/examples/quick_start_ios/mindspore-lite/Info.plist" 696 fi 697 698 # Exclude folder 699 MS_EXCLUDE_FOLDER "${workspace}/${REPOSITORY_NAME}" "${LOCAL_EXCLUDE_FOLDER}" 700 701 # Check tab 702 echo "tab scanning" 703 cd "${workspace}/${REPOSITORY_NAME}" || return 704 error_number=$(git grep -I -n $'\t' -- . ':(exclude)*.git*' ':(exclude)third_party' ':(exclude)**Makefile' ':(exclude)*.bin' ':(exclude)*.xml' ':(exclude)*.rst' ':(exclude)*.docx' ':(exclude)*.xlsx' ':(exclude)*.pdf' ':(exclude)*.mindir' ':(exclude)*.css' ':(exclude)*.js' ':(exclude)*.html' ':(exclude)*.patch' | wc -l) 705 if [ "${error_number}" -ne "0" ]; then 706 # Print content 707 echo "Problem items(The following files have tabs, please convert tab to space): " 708 git grep -I -n $'\t' -- . ':(exclude)*.git*' ':(exclude)third_party' ':(exclude)**Makefile' ':(exclude)*.bin' ':(exclude)*.xml' ':(exclude)*.rst' ':(exclude)*.docx' ':(exclude)*.xlsx' ':(exclude)*.pdf' ':(exclude)*.mindir' ':(exclude)*.css' ':(exclude)*.js' ':(exclude)*.html' ':(exclude)*.patch' 709 fi 710 DP_ASSERT_EQUAL "$error_number" "0" "Tab scanning" 711} 712 713# 按照门禁顺序进行检查 714clang_format_check 715cmakelint_check 716codespell_check 717cpplint_check 718lizard_check 719markdownlint_check 720pylint_check 721if [ "$os_name" == "Linux" ] || [ "$os_name" == "Darwi" ]; then 722 shellcheck_check 723fi 724tab_check 725 726if [ "$status" == 0 ]; then 727 echo "No error found during pre-push scanning stage, your code is ready for push!" 728else 729 echo "Total error number is $status, please correct your code before push!" 730fi 731exit $status 732