• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -- coding: utf-8 --
3# Copyright (c) 2022-2024 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import argparse
17import json
18import os
19import multiprocessing
20import sys
21import subprocess
22import re
23
24
25def get_args():
26    parser = argparse.ArgumentParser(
27        description="Runner for clang-tidy for panda project.")
28    parser.add_argument(
29        'panda_dir', help='panda sources directory.', type=str)
30    parser.add_argument(
31        'build_dir', help='panda build directory.', type=str)
32    parser.add_argument(
33        '--filename-filter', type=str, action='store', dest='filename_filter',
34        required=False, default="*",
35        help='Regexp for filename with path to it. If missed all source files will be checked.')
36    parser.add_argument(
37        '--full', action="store_true", help='Check all files with all compile keys.')
38    parser.add_argument(
39        '--check-libabckit', action="store_true", help='Check runtime_core/libabckit folder')
40    parser.add_argument(
41        '--header-filter', type=str, action='store', dest='header_filter',
42        required=False, default=".*",
43        help='Ignore header filter from .clang-tidy file.')
44    parser.add_argument(
45        '--proc-count', type=int, action='store', dest='proc_count',
46        required=False, default="-1",
47        help='Paralell process count of clang-tidy')
48
49    return parser.parse_args()
50
51
52default_disabled_checks = [
53    # aliases for other checks(here full list: https://clang.llvm.org/extra/clang-tidy/checks/list.html):
54    "-bugprone-narrowing-conversions",
55    "-cert-con36-c",
56    "-cert-con54-cpp",
57    "-cert-dcl03-c",
58    "-cert-dcl16-c",
59    "-cert-dcl37-c",
60    "-cert-dcl51-cpp",
61    "-cert-dcl54-cpp",
62    "-cert-dcl59-cpp",
63    "-cert-err09-cpp",
64    "-cert-err61-cpp",
65    "-cert-exp42-c",
66    "-cert-fio38-c",
67    "-cert-flp37-c",
68    "-cert-msc30-c",
69    "-cert-msc32-c",
70    "-cert-oop11-cpp",
71    "-cert-oop54-cpp",
72    "-cert-pos44-c",
73    "-cert-pos47-c",
74    "-cert-sig30-c",
75    "-cert-str34-c",
76    "-cppcoreguidelines-avoid-c-arrays",
77    "-cppcoreguidelines-avoid-magic-numbers",
78    "-cppcoreguidelines-c-copy-assignment-signature",
79    "-cppcoreguidelines-explicit-virtual-functions",
80    "-cppcoreguidelines-macro-to-enum",
81    "-cppcoreguidelines-non-private-member-variables-in-classes",
82    "-fuchsia-header-anon-namespaces",
83    # CHECKER_IGNORE_NEXTLINE(AF0010)
84    "-google-readability-braces-around-statements",
85    # CHECKER_IGNORE_NEXTLINE(AF0010)
86    "-google-readability-function-size",
87    # CHECKER_IGNORE_NEXTLINE(AF0010)
88    "-google-readability-namespace-comments",
89    "-hicpp-avoid-c-arrays",
90    "-hicpp-avoid-goto",
91    "-hicpp-braces-around-statements",
92    "-hicpp-deprecated-headers",
93    "-hicpp-explicit-conversions",
94    "-hicpp-function-size",
95    "-hicpp-invalid-access-moved",
96    "-hicpp-member-init",
97    "-hicpp-move-const-arg",
98    "-hicpp-named-parameter",
99    "-hicpp-new-delete-operators",
100    "-hicpp-no-array-decay",
101    "-hicpp-no-malloc",
102    "-hicpp-noexcept-move",
103    "-hicpp-special-member-functions",
104    "-hicpp-static-assert",
105    "-hicpp-undelegated-constructor",
106    "-hicpp-uppercase-literal-suffix",
107    "-hicpp-use-auto",
108    "-hicpp-use-emplace",
109    "-hicpp-use-equals-default",
110    "-hicpp-use-equals-delete",
111    "-hicpp-use-noexcept",
112    "-hicpp-use-nullptr",
113    "-hicpp-use-override",
114    "-hicpp-vararg",
115    "-llvm-else-after-return",
116    "-llvm-qualified-auto",
117    # explicitly disabled checks
118    # disabled because it is hard to write macros with types with it
119    "-bugprone-macro-parentheses",
120    # disabled because of incorrect root prefix
121    "-llvm-header-guard",
122    # disabled because conflicts with the clang-format
123    "-llvm-include-order",
124    # disabled to use non-const references
125    # CHECKER_IGNORE_NEXTLINE(AF0010)
126    "-google-runtime-references",
127    # disabled because we have a lot of false positives and it is stylistic check
128    "-fuchsia-trailing-return",
129    # disabled because we use functions with default arguments a lot
130    "-fuchsia-default-arguments-calls",
131    # disabled because we use functions with default arguments a lot
132    "-fuchsia-default-arguments-declarations",
133    # disabled as a stylistic check
134    "-modernize-use-trailing-return-type",
135    # Fix all occurences
136    "-readability-static-accessed-through-instance",
137    # Fix all occurences
138    "-bugprone-sizeof-expression",
139    # Fix all occurences
140    "-readability-convert-member-functions-to-static",
141    # Fix all occurences
142    "-bugprone-branch-clone",
143    # disabled llvm-libc specific checks
144    "-llvmlibc-*",
145    # disabled FGPA specific checks
146    "-altera-*",
147    # disable all suggestions to use abseil library
148    "-abseil-*",
149    # disabled as a stylistic check
150    "-readability-identifier-length",
151    # Look if we want to use GSL or gsl-lite
152    "-cppcoreguidelines-owning-memory",
153    # Look into issue with ASSERT
154    "-cppcoreguidelines-pro-bounds-array-to-pointer-decay",
155    # Look if we want to use GSL or gsl-lite
156    "-cppcoreguidelines-pro-bounds-constant-array-index",
157    # Consider to remove from global list
158    "-cppcoreguidelines-pro-type-const-cast",
159    # Consider to remove from global list
160    "-cppcoreguidelines-pro-type-reinterpret-cast",
161    # Look into it
162    "-cppcoreguidelines-pro-type-static-cast-downcast",
163    # Look into it
164    "-fuchsia-default-arguments",
165    # Consider to remove from global list
166    "-fuchsia-overloaded-operator",
167    # Look into it
168    "-modernize-use-nodiscard",
169    "-cert-dcl50-cpp",
170    # candidates for removal:
171    # For some reason become failed in DEFAULT_MOVE_SEMANTIC
172    "-performance-noexcept-move-constructor",
173
174    # clang-14 temporary exceptions
175    "-bugprone-easily-swappable-parameters",
176    "-bugprone-reserved-identifier",
177    "-bugprone-signed-char-misuse",
178    "-bugprone-implicit-widening-of-multiplication-result",
179    "-bugprone-suspicious-include",
180    "-bugprone-dynamic-static-initializers",
181
182    "-cppcoreguidelines-avoid-non-const-global-variables",
183    "-cppcoreguidelines-virtual-class-destructor",
184    "-cppcoreguidelines-prefer-member-initializer",
185    "-cppcoreguidelines-init-variables",
186    "-cppcoreguidelines-narrowing-conversions",
187
188    # CHECKER_IGNORE_NEXTLINE(AF0010)
189    "-google-upgrade-googletest-case",
190
191    "-readability-redundant-access-specifiers",
192    "-readability-qualified-auto",
193    "-readability-make-member-function-const",
194    "-readability-container-data-pointer",
195    "-readability-function-cognitive-complexity",
196    "-readability-use-anyofallof",
197    "-readability-suspicious-call-argument",
198
199    "-modernize-return-braced-init-list",
200
201    "-cert-err33-c",
202
203    # CHECKER_IGNORE_NEXTLINE(AF0010)
204    "-google-readability-casting",
205
206    "-concurrency-mt-unsafe",
207    "-performance-no-int-to-ptr",
208    "-misc-no-recursion",
209
210    # CHECKER_IGNORE_NEXTLINE(AF0010)
211    "-google-readability-avoid-underscore-in-googletest-name",
212    "-readability-avoid-const-params-in-decls"
213]
214
215
216def run_clang_tidy(src_path: str, config_file_path: str, build_dir: str, header_filter: str, compile_args: str) -> bool:
217    # Used by ctcache to provide a wrapper for real clang-tidy that will check the cache
218    # before launching clang-tidy and save the result to ctcache server
219    cmd_path = os.getenv('CLANG_TIDY_PATH')
220    if not cmd_path:
221        cmd_path = 'clang-tidy-14'
222    cmd = [cmd_path]
223    cmd += ['-checks=*,' + ','.join(default_disabled_checks)]
224    cmd += ['--header-filter=' + header_filter]
225    cmd += ['--config-file=' + os.path.join(config_file_path, '.clang-tidy')]
226    cmd += [src_path]
227    cmd += ['--']
228    cmd += compile_args.split()
229
230    try:
231        subprocess.check_output(cmd, cwd=build_dir, stderr=subprocess.STDOUT)
232    except subprocess.CalledProcessError as e:
233        # Skip error for some invalid release configurations.
234        if not e.stdout:
235            print("Note: missed output for ", src_path)
236            return True
237
238        out_msg = e.stdout.decode()
239        if ',-warnings-as-errors]' not in out_msg:
240            print("Note: bad output for ", src_path)
241            return True
242
243        print('Failed: ' + ' '.join(cmd) + '\n' + out_msg)
244
245        if e.stderr:
246            print(e.stderr.decode())
247
248        return False
249
250    return True
251
252
253def get_full_path(relative_path: str, location_base: str, panda_dir: str, build_dir: str) -> str:
254    full_path = panda_dir if location_base == "PANDA_DIR" else build_dir
255    full_path += "/" + relative_path
256    full_path = str(os.path.realpath(full_path))
257    return full_path
258
259
260def check_file_list(file_list: list, panda_dir: str, build_dir: str, header_filter: str, proc_count: int) -> bool:
261    pool = multiprocessing.Pool(proc_count)
262    jobs = []
263    for src, args in file_list:
264
265        msg = "Done clang-tidy: %s" % (src)
266
267        proc = pool.apply_async(func=run_clang_tidy, args=(
268            src, panda_dir, build_dir, header_filter, args))
269        jobs.append((proc, msg))
270
271    # Wait for jobs to complete before exiting
272    total_count = str(len(jobs))
273    main_ret_val = True
274    idx = 0
275    while jobs:
276        upd_job = []
277        for proc, msg in jobs:
278            if not proc.ready():
279                upd_job.append((proc, msg))
280                continue
281
282            idx += 1
283            print("[%s/%s] %s" % (str(idx), total_count, msg))
284            if main_ret_val and not proc.get():
285                main_ret_val = False
286
287        jobs = upd_job
288
289    # Safely terminate the pool
290    pool.close()
291    pool.join()
292
293    return main_ret_val
294
295
296def need_to_ignore_file(file_path: str, panda_dir: str, build_dir: str) -> bool:
297    src_exts = (".c", '.cc', ".cp", ".cxx", ".cpp", ".CPP", ".c++", ".C")
298    if not file_path.endswith(src_exts):
299        return True
300
301    # Skip third_party.
302    regexp = re.compile(".*/third_party/.*")
303    if regexp.search(file_path):
304        return True
305
306    return False
307
308
309def get_file_list(panda_dir: str, build_dir: str, filename_filter: str) -> list:
310    json_cmds_path = os.path.join(build_dir, 'compile_commands.json')
311    cmds_json = []
312    with open(json_cmds_path, 'r') as f:
313        cmds_json = json.load(f)
314
315    if not cmds_json:
316        return []
317
318    regexp = None
319    if filename_filter != '*':
320        regexp = re.compile(filename_filter)
321
322    file_list = []
323    for cmd in cmds_json:
324        # this check is needed to exclude symlinks in third_party
325        file_path_raw = cmd["file"]
326        if need_to_ignore_file(file_path_raw, panda_dir, build_dir):
327            continue
328
329        file_path = str(os.path.realpath(cmd["file"]))
330        if need_to_ignore_file(file_path, panda_dir, build_dir):
331            continue
332
333        if regexp and not regexp.search(file_path):
334            continue
335
336        compile_args = cmd["command"]
337        # strip unnecessary escape sequences
338        compile_args = compile_args.replace('\\', '')
339        # Removing sysroot for correct clang-tidy work on cross-compiled arm
340        sysroot = re.search(r' --sysroot[\w\d\S]+', compile_args)
341        if sysroot:
342            compile_args = compile_args.replace(sysroot.group(0), '')
343
344        file_list.append((file_path, compile_args))
345
346    file_list.sort(key=lambda tup: tup[0])
347    return file_list
348
349
350# Remove noisy test files duplicates.
351def filter_test_file_duplicates(file_list: list) -> list:
352    filtered = []
353    regexp = re.compile(".*/tests/.*")
354    for file_path, keys in file_list:
355        if not filtered:
356            filtered.append((file_path, keys))
357            continue
358
359        if filtered[-1][0] != file_path:
360            filtered.append((file_path, keys))
361            continue
362
363        if not regexp.search(file_path):
364            filtered.append((file_path, keys))
365
366    filtered.sort(key=lambda tup: tup[0])
367    return filtered;
368
369
370def check_file_list_duplicate(dup_path : str, file_list: list) -> bool:
371    count = 0
372    for file_path, keys in file_list:
373        if dup_path == file_path:
374            count += 1
375
376        if count > 1:
377            return True
378
379    return False
380
381
382# Remove same files if their compile keys has minor differents.
383def filter_file_duplicated_options(file_list: list) -> list:
384    filtered = []
385    regexp = re.compile(".*DPANDA_ENABLE_RELAYOUT_PROFILE.*")
386    for file_path, keys in file_list:
387        if not check_file_list_duplicate(file_path, file_list):
388            filtered.append((file_path, keys))
389            continue
390
391        if not regexp.search(keys):
392            filtered.append((file_path, keys))
393
394    filtered.sort(key=lambda tup: tup[0])
395    return filtered;
396
397
398def filter_file_list(file_list: list) -> list:
399    print('Files before filter:', len(file_list))
400
401    filtered = filter_file_duplicated_options(file_list)
402    filtered = filter_test_file_duplicates(filtered)
403
404    print('Filtered files:', len(filtered))
405    return filtered
406
407
408def verify_uniq_element_list(uniq_element_list: list) -> bool:
409    return len(uniq_element_list) == len(set(uniq_element_list))
410
411
412def verify_args(panda_dir: str, build_dir: str) -> str:
413    if not verify_uniq_element_list(default_disabled_checks):
414        return "Error: Dupclicated defauls disabled checks"
415
416    return ""
417
418
419def check_headers_in_es2panda_sources(panda_dir):
420    es2panda_dir = os.path.join(panda_dir, "tools/es2panda")
421    result = []
422    for root, dirs, files in os.walk(es2panda_dir):
423        for file in files:
424            file_path = os.path.join(root, file)
425            _, extension = os.path.splitext(file_path)
426            if extension != ".h" and extension != ".cpp":
427                continue
428            with open(file_path, "r") as source_file:
429                for line in source_file.readlines():
430                    line = line.replace(" ", "")
431                    if (line.startswith("#include\"tools/es2panda")):
432                        result.append(f"Error: use of header starting with tools/es2panda in {file_path}")
433                        continue
434    if len(result) > 0:
435        for file in result:
436            print(file)
437        sys.exit(1)
438
439
440def check_file_list_for_system_headers_includes(file_list: list):
441    system_headers_ = []
442    regexp = re.compile("-I[^ ]*/third_party[^ ]*")
443    for path, compile_args in file_list:
444        match = regexp.search(compile_args)
445        if match:
446            system_headers_.append((path, match.group()))
447
448    return system_headers_
449
450
451def get_proc_count(cmd_ard : int) -> int:
452    if cmd_ard > 0:
453        return cmd_ard
454
455    min_proc_str = os.getenv('NPROC_PER_JOB')
456    if min_proc_str:
457        return int(min_proc_str)
458
459    return multiprocessing.cpu_count()
460
461if __name__ == "__main__":
462    arguments = get_args()
463    files_list = []
464
465    if not os.path.exists(os.path.join(arguments.build_dir, 'compile_commands.json')):
466        sys.exit("Error: Missing file `compile_commands.json` in build directory")
467
468    # A lot of false positive checks on GTESTS in libabckit
469    if arguments.check_libabckit:
470        default_disabled_checks.append("-fuchsia-statically-constructed-objects")
471        default_disabled_checks.append("-cert-err58-cpp")
472
473    err_msg = verify_args(arguments.panda_dir, arguments.build_dir)
474    if err_msg:
475        sys.exit(err_msg)
476
477    files_list = get_file_list(
478        arguments.panda_dir, arguments.build_dir, arguments.filename_filter)
479
480    if not files_list:
481        sys.exit("Can't be prepaired source list."
482                 "Please check availble in build `dir compile_commands.json`"
483                 "and correcting of parameter `--filename-filter` if you use it.")
484
485    if not arguments.check_libabckit:
486        check_headers_in_es2panda_sources(arguments.panda_dir)
487        print('Checked for system headers: Starting')
488        system_headers = check_file_list_for_system_headers_includes(files_list)
489        if system_headers:
490            err_msg = "Error: third_party includes should be marked as system\n"
491            for e_path, e_system_header in system_headers:
492                err_msg += e_path + " error: " + e_system_header + "\n"
493            sys.exit(err_msg)
494        print('Checked for system headers: Done')
495
496    if not arguments.full:
497        files_list = filter_file_list(files_list)
498    else:
499        # Disable ctcache in full mode to handle cases when caching works incorrectly
500        os.environ['CTCACHE_DISABLE'] = '1'
501
502    process_count = get_proc_count(arguments.proc_count)
503    print('clang-tidy proc_count: ' + str(process_count))
504    conf_file_path = arguments.panda_dir
505    if arguments.check_libabckit:
506        conf_file_path = os.path.join(conf_file_path, "libabckit")
507    if not check_file_list(files_list, conf_file_path, arguments.build_dir, arguments.header_filter, process_count):
508        sys.exit("Failed: Clang-tidy get errors")
509
510    print("Clang-tidy was passed successfully!")
511