1#!/usr/bin/env python3 2# -- coding: utf-8 -- 3# Copyright (c) 2022-2024 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import argparse 17import json 18import os 19import multiprocessing 20import sys 21import subprocess 22import re 23 24 25def get_args(): 26 parser = argparse.ArgumentParser( 27 description="Runner for clang-tidy for panda project.") 28 parser.add_argument( 29 'panda_dir', help='panda sources directory.', type=str) 30 parser.add_argument( 31 'build_dir', help='panda build directory.', type=str) 32 parser.add_argument( 33 '--filename-filter', type=str, action='store', dest='filename_filter', 34 required=False, default="*", 35 help='Regexp for filename with path to it. If missed all source files will be checked.') 36 parser.add_argument( 37 '--full', action="store_true", help='Check all files with all compile keys.') 38 parser.add_argument( 39 '--proc-count', type=int, action='store', dest='proc_count', 40 required=False, default="-1", 41 help='Paralell process count of clang-tidy') 42 43 return parser.parse_args() 44 45 46default_disabled_checks = [ 47 # aliases for other checks(here full list: https://clang.llvm.org/extra/clang-tidy/checks/list.html): 48 "-bugprone-narrowing-conversions", 49 "-cert-con36-c", 50 "-cert-con54-cpp", 51 "-cert-dcl03-c", 52 "-cert-dcl16-c", 53 "-cert-dcl37-c", 54 "-cert-dcl51-cpp", 55 "-cert-dcl54-cpp", 56 "-cert-dcl59-cpp", 57 "-cert-err09-cpp", 58 "-cert-err61-cpp", 59 "-cert-exp42-c", 60 "-cert-fio38-c", 61 "-cert-flp37-c", 62 "-cert-msc30-c", 63 "-cert-msc32-c", 64 "-cert-oop11-cpp", 65 "-cert-oop54-cpp", 66 "-cert-pos44-c", 67 "-cert-pos47-c", 68 "-cert-sig30-c", 69 "-cert-str34-c", 70 "-cppcoreguidelines-avoid-c-arrays", 71 "-cppcoreguidelines-avoid-magic-numbers", 72 "-cppcoreguidelines-c-copy-assignment-signature", 73 "-cppcoreguidelines-explicit-virtual-functions", 74 "-cppcoreguidelines-macro-to-enum", 75 "-cppcoreguidelines-non-private-member-variables-in-classes", 76 "-fuchsia-header-anon-namespaces", 77 # CHECKER_IGNORE_NEXTLINE(AF0010) 78 "-google-readability-braces-around-statements", 79 # CHECKER_IGNORE_NEXTLINE(AF0010) 80 "-google-readability-function-size", 81 # CHECKER_IGNORE_NEXTLINE(AF0010) 82 "-google-readability-namespace-comments", 83 "-hicpp-avoid-c-arrays", 84 "-hicpp-avoid-goto", 85 "-hicpp-braces-around-statements", 86 "-hicpp-deprecated-headers", 87 "-hicpp-explicit-conversions", 88 "-hicpp-function-size", 89 "-hicpp-invalid-access-moved", 90 "-hicpp-member-init", 91 "-hicpp-move-const-arg", 92 "-hicpp-named-parameter", 93 "-hicpp-new-delete-operators", 94 "-hicpp-no-array-decay", 95 "-hicpp-no-malloc", 96 "-hicpp-noexcept-move", 97 "-hicpp-special-member-functions", 98 "-hicpp-static-assert", 99 "-hicpp-undelegated-constructor", 100 "-hicpp-uppercase-literal-suffix", 101 "-hicpp-use-auto", 102 "-hicpp-use-emplace", 103 "-hicpp-use-equals-default", 104 "-hicpp-use-equals-delete", 105 "-hicpp-use-noexcept", 106 "-hicpp-use-nullptr", 107 "-hicpp-use-override", 108 "-hicpp-vararg", 109 "-llvm-else-after-return", 110 "-llvm-qualified-auto", 111 # explicitly disabled checks 112 # disabled because it is hard to write macros with types with it 113 "-bugprone-macro-parentheses", 114 # disabled because of incorrect root prefix 115 "-llvm-header-guard", 116 # disabled because conflicts with the clang-format 117 "-llvm-include-order", 118 # disabled to use non-const references 119 # CHECKER_IGNORE_NEXTLINE(AF0010) 120 "-google-runtime-references", 121 # disabled because we have a lot of false positives and it is stylistic check 122 "-fuchsia-trailing-return", 123 # disabled because we use functions with default arguments a lot 124 "-fuchsia-default-arguments-calls", 125 # disabled because we use functions with default arguments a lot 126 "-fuchsia-default-arguments-declarations", 127 # disabled as a stylistic check 128 "-modernize-use-trailing-return-type", 129 # Fix all occurences 130 "-readability-static-accessed-through-instance", 131 # Fix all occurences 132 "-bugprone-sizeof-expression", 133 # Fix all occurences 134 "-readability-convert-member-functions-to-static", 135 # Fix all occurences 136 "-bugprone-branch-clone", 137 # disabled llvm-libc specific checks 138 "-llvmlibc-*", 139 # disabled FGPA specific checks 140 "-altera-*", 141 # disable all suggestions to use abseil library 142 "-abseil-*", 143 # disabled as a stylistic check 144 "-readability-identifier-length", 145 # Look if we want to use GSL or gsl-lite 146 "-cppcoreguidelines-owning-memory", 147 # Look into issue with ASSERT 148 "-cppcoreguidelines-pro-bounds-array-to-pointer-decay", 149 # Look if we want to use GSL or gsl-lite 150 "-cppcoreguidelines-pro-bounds-constant-array-index", 151 # Consider to remove from global list 152 "-cppcoreguidelines-pro-type-const-cast", 153 # Consider to remove from global list 154 "-cppcoreguidelines-pro-type-reinterpret-cast", 155 # Look into it 156 "-cppcoreguidelines-pro-type-static-cast-downcast", 157 # Look into it 158 "-fuchsia-default-arguments", 159 # Consider to remove from global list 160 "-fuchsia-overloaded-operator", 161 # Look into it 162 "-modernize-use-nodiscard", 163 "-cert-dcl50-cpp", 164 # candidates for removal: 165 # For some reason become failed in DEFAULT_MOVE_SEMANTIC 166 "-performance-noexcept-move-constructor", 167 168 # clang-14 temporary exceptions 169 "-bugprone-easily-swappable-parameters", 170 "-bugprone-reserved-identifier", 171 "-bugprone-signed-char-misuse", 172 "-bugprone-implicit-widening-of-multiplication-result", 173 "-bugprone-suspicious-include", 174 "-bugprone-dynamic-static-initializers", 175 176 "-cppcoreguidelines-avoid-non-const-global-variables", 177 "-cppcoreguidelines-virtual-class-destructor", 178 "-cppcoreguidelines-prefer-member-initializer", 179 "-cppcoreguidelines-init-variables", 180 "-cppcoreguidelines-narrowing-conversions", 181 182 # CHECKER_IGNORE_NEXTLINE(AF0010) 183 "-google-upgrade-googletest-case", 184 185 "-readability-redundant-access-specifiers", 186 "-readability-qualified-auto", 187 "-readability-make-member-function-const", 188 "-readability-container-data-pointer", 189 "-readability-function-cognitive-complexity", 190 "-readability-use-anyofallof", 191 "-readability-suspicious-call-argument", 192 193 "-modernize-return-braced-init-list", 194 195 "-cert-err33-c", 196 197 # CHECKER_IGNORE_NEXTLINE(AF0010) 198 "-google-readability-casting", 199 200 "-concurrency-mt-unsafe", 201 "-performance-no-int-to-ptr", 202 "-misc-no-recursion", 203 204 # CHECKER_IGNORE_NEXTLINE(AF0010) 205 "-google-readability-avoid-underscore-in-googletest-name", 206 "-readability-avoid-const-params-in-decls" 207] 208 209 210def run_clang_tidy(src_path: str, panda_dir: str, build_dir: str, compile_args: str) -> bool: 211 # Used by ctcache to provide a wrapper for real clang-tidy that will check the cache 212 # before launching clang-tidy and save the result to ctcache server 213 cmd_path = os.getenv('CLANG_TIDY_PATH') 214 if not cmd_path: 215 cmd_path = 'clang-tidy-14' 216 cmd = [cmd_path] 217 cmd += ['-checks=*,' + ','.join(default_disabled_checks)] 218 cmd += ['--header-filter=.*'] 219 cmd += ['--config-file=' + os.path.join(panda_dir, '.clang-tidy')] 220 cmd += [src_path] 221 cmd += ['--'] 222 cmd += compile_args.split() 223 224 try: 225 subprocess.check_output(cmd, cwd=build_dir, stderr=subprocess.STDOUT) 226 except subprocess.CalledProcessError as e: 227 # Skip error for some invalid release configurations. 228 if not e.stdout: 229 print("Note: missed output for ", src_path) 230 return True 231 232 out_msg = e.stdout.decode() 233 if ',-warnings-as-errors]' not in out_msg: 234 print("Note: bad output for ", src_path) 235 return True 236 237 print('Failed: ' + ' '.join(cmd) + '\n' + out_msg) 238 239 if e.stderr: 240 print(e.stderr.decode()) 241 242 return False 243 244 return True 245 246 247def get_full_path(relative_path: str, location_base: str, panda_dir: str, build_dir: str) -> str: 248 full_path = panda_dir if location_base == "PANDA_DIR" else build_dir 249 full_path += "/" + relative_path 250 full_path = str(os.path.realpath(full_path)) 251 return full_path 252 253 254def check_file_list(file_list: list, panda_dir: str, build_dir: str, proc_count: int) -> bool: 255 pool = multiprocessing.Pool(proc_count) 256 jobs = [] 257 for src, args in file_list: 258 259 msg = "Done clang-tidy: %s" % (src) 260 261 proc = pool.apply_async(func=run_clang_tidy, args=( 262 src, panda_dir, build_dir, args)) 263 jobs.append((proc, msg)) 264 265 # Wait for jobs to complete before exiting 266 total_count = str(len(jobs)) 267 main_ret_val = True 268 idx = 0 269 while jobs: 270 upd_job = [] 271 for proc, msg in jobs: 272 if not proc.ready(): 273 upd_job.append((proc, msg)) 274 continue 275 276 idx += 1 277 print("[%s/%s] %s" % (str(idx), total_count, msg)) 278 if main_ret_val and not proc.get(): 279 main_ret_val = False 280 281 jobs = upd_job 282 283 # Safely terminate the pool 284 pool.close() 285 pool.join() 286 287 return main_ret_val 288 289 290def need_to_ignore_file(file_path: str, panda_dir: str, build_dir: str) -> bool: 291 src_exts = (".c", '.cc', ".cp", ".cxx", ".cpp", ".CPP", ".c++", ".C") 292 if not file_path.endswith(src_exts): 293 return True 294 295 # Skip third_party. 296 regexp = re.compile(".*/third_party/.*") 297 if regexp.search(file_path): 298 return True 299 300 return False 301 302 303def get_file_list(panda_dir: str, build_dir: str, filename_filter: str) -> list: 304 json_cmds_path = os.path.join(build_dir, 'compile_commands.json') 305 cmds_json = [] 306 with open(json_cmds_path, 'r') as f: 307 cmds_json = json.load(f) 308 309 if not cmds_json: 310 return [] 311 312 regexp = None 313 if filename_filter != '*': 314 regexp = re.compile(filename_filter) 315 316 file_list = [] 317 for cmd in cmds_json: 318 file_path = str(os.path.realpath(cmd["file"])) 319 if need_to_ignore_file(file_path, panda_dir, build_dir): 320 continue 321 322 if regexp and not regexp.search(file_path): 323 continue 324 325 compile_args = cmd["command"] 326 # strip unnecessary escape sequences 327 compile_args = compile_args.replace('\\', '') 328 # Removing sysroot for correct clang-tidy work on cross-compiled arm 329 sysroot = re.search(r' --sysroot[\w\d\S]+', compile_args) 330 if sysroot: 331 compile_args = compile_args.replace(sysroot.group(0), '') 332 333 file_list.append((file_path, compile_args)) 334 335 file_list.sort(key=lambda tup: tup[0]) 336 return file_list 337 338 339# Remove noisy test files duplicates. 340def filter_test_file_duplicates(file_list: list) -> list: 341 filtered = [] 342 regexp = re.compile(".*/tests/.*") 343 for file_path, keys in file_list: 344 if not filtered: 345 filtered.append((file_path, keys)) 346 continue 347 348 if filtered[-1][0] != file_path: 349 filtered.append((file_path, keys)) 350 continue 351 352 if not regexp.search(file_path): 353 filtered.append((file_path, keys)) 354 355 filtered.sort(key=lambda tup: tup[0]) 356 return filtered; 357 358 359def check_file_list_duplicate(dup_path : str, file_list: list) -> bool: 360 count = 0 361 for file_path, keys in file_list: 362 if dup_path == file_path: 363 count += 1 364 365 if count > 1: 366 return True 367 368 return False 369 370 371# Remove same files if their compile keys has minor differents. 372def filter_file_duplicated_options(file_list: list) -> list: 373 filtered = [] 374 regexp = re.compile(".*DPANDA_ENABLE_RELAYOUT_PROFILE.*") 375 for file_path, keys in file_list: 376 if not check_file_list_duplicate(file_path, file_list): 377 filtered.append((file_path, keys)) 378 continue 379 380 if not regexp.search(keys): 381 filtered.append((file_path, keys)) 382 383 filtered.sort(key=lambda tup: tup[0]) 384 return filtered; 385 386 387def filter_file_list(file_list: list) -> list: 388 print('Files before filter:', len(file_list)) 389 390 filtered = filter_file_duplicated_options(file_list) 391 filtered = filter_test_file_duplicates(filtered) 392 393 print('Filtered files:', len(filtered)) 394 return filtered 395 396 397def verify_uniq_element_list(uniq_element_list: list) -> bool: 398 return len(uniq_element_list) == len(set(uniq_element_list)) 399 400 401def verify_args(panda_dir: str, build_dir: str) -> str: 402 if not verify_uniq_element_list(default_disabled_checks): 403 return "Error: Dupclicated defauls disabled checks" 404 405 return "" 406 407 408def check_headers_in_es2panda_sources(panda_dir): 409 es2panda_dir = os.path.join(panda_dir, "tools/es2panda") 410 result = [] 411 for root, dirs, files in os.walk(es2panda_dir): 412 for file in files: 413 file_path = os.path.join(root, file) 414 _, extension = os.path.splitext(file_path) 415 if extension != ".h" and extension != ".cpp": 416 continue 417 with open(file_path, "r") as source_file: 418 for line in source_file.readlines(): 419 line = line.replace(" ", "") 420 if (line.startswith("#include\"tools/es2panda")): 421 result.append(f"Error: use of header starting with tools/es2panda in {file_path}") 422 continue 423 if len(result) > 0: 424 for file in result: 425 print(file) 426 sys.exit(1) 427 428 429def check_file_list_for_system_headers_includes(file_list: list): 430 system_headers_ = [] 431 regexp = re.compile("-I[^ ]*/third_party[^ ]*") 432 for path, compile_args in file_list: 433 match = regexp.search(compile_args) 434 if match: 435 system_headers_.append((path, match.group())) 436 437 return system_headers_ 438 439 440def get_proc_count(cmd_ard : int) -> int: 441 if cmd_ard > 0: 442 return cmd_ard 443 444 min_proc_str = os.getenv('NPROC_PER_JOB') 445 if min_proc_str: 446 return int(min_proc_str) 447 448 return multiprocessing.cpu_count() 449 450if __name__ == "__main__": 451 arguments = get_args() 452 files_list = [] 453 454 if not os.path.exists(os.path.join(arguments.build_dir, 'compile_commands.json')): 455 sys.exit("Error: Missing file `compile_commands.json` in build directory") 456 457 err_msg = verify_args(arguments.panda_dir, arguments.build_dir) 458 if err_msg: 459 sys.exit(err_msg) 460 461 files_list = get_file_list( 462 arguments.panda_dir, arguments.build_dir, arguments.filename_filter) 463 464 if not files_list: 465 sys.exit("Can't be prepaired source list." 466 "Please check availble in build `dir compile_commands.json`" 467 "and correcting of parameter `--filename-filter` if you use it.") 468 469 check_headers_in_es2panda_sources(arguments.panda_dir) 470 print('Checked for system headers: Starting') 471 system_headers = check_file_list_for_system_headers_includes(files_list) 472 if system_headers: 473 err_msg = "Error: third_party includes should be marked as system\n" 474 for e_path, e_system_header in system_headers: 475 err_msg += e_path + " error: " + e_system_header + "\n" 476 sys.exit(err_msg) 477 print('Checked for system headers: Done') 478 479 if not arguments.full: 480 files_list = filter_file_list(files_list) 481 else: 482 # Disable ctcache in full mode to handle cases when caching works incorrectly 483 os.environ['CTCACHE_DISABLE'] = '1' 484 485 process_count = get_proc_count(arguments.proc_count) 486 print('clang-tidy proc_count: ' + str(process_count)) 487 if not check_file_list(files_list, arguments.panda_dir, arguments.build_dir, process_count): 488 sys.exit("Failed: Clang-tidy get errors") 489 490 print("Clang-tidy was passed successfully!") 491