Lines Matching +full:output +full:- +full:files
4 # Use of this source code is governed by a BSD-style license that can be
7 """Runs presubmit checks against a bundle of files."""
41 # listed here because (1) only very few files are actually type checked,
50 # - never write to stdout/stderr or read from stdin directly
51 # - return either a CheckResult, or a list of [(subcheck_name, CheckResult)]
52 # - ideally use thread_pool to check things concurrently
53 # - though it's important to note that these *also* live on the threadpool
62 ("output", str),
72 # The files and directories on which we run the mypy typechecker. The paths are
73 # relative to the root of the toolchain-utils repository.
84 "toolchain_utils_githooks/check-presubmit.py",
92 ) -> Tuple[int, str]:
102 encoding="utf-8",
108 def has_executable_on_path(exe: str) -> bool:
113 def remove_deleted_files(files: Iterable[str]) -> List[str]:
114 return [f for f in files if os.path.exists(f)]
117 def is_file_executable(file_path: str) -> bool:
122 # toolchain-utils/. Add that to PYTHONPATH to ensure that things like `cros
124 def env_with_pythonpath(toolchain_utils_root: str) -> Dict[str, str]:
143 def get_mypy() -> Optional[MyPyInvocation]:
154 - the command to invoke mypy, and
155 - any environment variables to set when invoking mypy
164 def get_from_pip() -> Optional[MyPyInvocation]:
165 rc, output = run_command_unchecked(pip + ["show", "mypy"])
169 m = re.search(r"^Location: (.*)", output, re.MULTILINE)
177 "-m",
189 subprocess.check_call(pip + ["install", "--user", "mypy"])
194 def get_pip() -> Optional[List[str]]:
205 subprocess.check_call(["python", "-m", "ensurepip"])
209 return ["python", "-m", "pip"]
215 ) -> CheckResult:
225 output="Check exited with an unexpected exception:\n%s"
233 ) -> CheckResult:
242 output="isort not found; skipping",
246 config_file_flag = f"--settings-file={config_file}"
247 command = [str(isort), "-c", config_file_flag] + list(python_files)
252 # isort fails when files have broken formatting.
256 output="",
273 output=f"`{shlex.join(command)}` failed; stdout/stderr:\n"
281 output="The following file(s) have formatting errors: %s" % bad_files,
288 ) -> CheckResult:
293 command: Command = [black, "--version"]
300 output="Failed getting black version; "
306 black_invocation: List[str] = [str(black), "--line-length=80"]
307 command = black_invocation + ["--check"] + list(python_files)
311 # black fails when files are poorly formatted.
315 output=f"Using {black_version!r}, no issues were found.",
319 # Output format looks something like:
326 output=f"Unparseable `black` output:\n{stdout_and_stderr}",
342 output=f"Unparseable `black` output:\n{stdout_and_stderr}",
354 output=f"Using {black_version!r} had the following errors:\n"
362 output=f"Using {black_version!r}, these file(s) have formatting "
371 files: Iterable[str],
372 ) -> CheckResult:
382 cmd = mypy.command + ["--version"]
383 exit_code, output = run_command_unchecked(
389 output=f"Failed getting mypy version; stdstreams: {output}",
392 # Prefix output with the version information.
393 prefix = f"Using {output.strip()}, "
395 cmd = mypy.command + ["--follow-imports=silent"] + list(files)
396 exit_code, output = run_command_unchecked(
402 output=f"{output}{prefix}checks passed",
408 output=f"{output}{prefix}type errors were found",
413 def check_python_file_headers(python_files: Iterable[str]) -> CheckResult:
420 with open(python_file, encoding="utf-8") as f:
431 output = []
433 output.append(
434 "The following files have no #!, but need one: %s" % add_hashbang
436 autofix.append(["sed", "-i", "1i#!/usr/bin/env python3"] + add_hashbang)
439 output.append(
440 "The following files have a #!, but shouldn't: %s" % remove_hashbang
442 autofix.append(["sed", "-i", "1d"] + remove_hashbang)
444 if not output:
447 output="",
452 output="\n".join(output),
460 files: Iterable[str],
461 ) -> CheckResults:
462 """Runs black on files to check for style bugs. Also checks for #!s."""
467 output="black isn't available on your $PATH. Please either "
472 python_files = [f for f in remove_deleted_files(files) if f.endswith(".py")]
476 output="no python files to check",
501 def file_is_relative_to(file: Path, potential_parent: Path) -> bool:
510 def is_file_in_any_of(file: Path, files_and_dirs: List[Path]) -> bool:
531 files: Iterable[str],
532 ) -> CheckResults:
533 """Runs static type checking for files in MYPY_CHECKED_FILES."""
538 for x in files
545 output="no python files to typecheck",
553 output="mypy not found. Please either enter a chroot "
569 def find_chromeos_root_directory() -> Optional[str]:
576 files: Iterable[str],
577 ) -> CheckResults:
585 def try_run_cros_lint(cros_binary: str) -> Optional[CheckResult]:
586 exit_code, output = run_command_unchecked(
587 [cros_binary, "lint", "--"] + list(files),
599 output=output,
617 def check_result_from_command(command: List[str]) -> CheckResult:
618 exit_code, output = run_command_unchecked(
623 output=output,
627 python_files = [f for f in remove_deleted_files(files) if f.endswith(".py")]
630 def run_pylint() -> CheckResult:
636 go_files = [f for f in remove_deleted_files(files) if f.endswith(".go")]
639 def run_golint() -> CheckResult:
642 ["golint", "-set_exit_status"] + go_files
652 output=complaint,
669 output=complaint,
675 angry_complaint = (complaint + "\n\n" + angry_result.output).strip()
676 results[0] = (name, angry_result._replace(output=angry_complaint))
680 def check_go_format(toolchain_utils_root, _thread_pool, files): argument
681 """Runs gofmt on files to check for style bugs."""
686 output="gofmt isn't available on your $PATH. Please either "
691 go_files = [f for f in remove_deleted_files(files) if f.endswith(".go")]
695 output="no go files to check",
699 command = [gofmt, "-l"] + go_files
700 exit_code, output = run_command_unchecked(command, cwd=toolchain_utils_root)
705 output=f"{shlex.join(command)} failed; stdout/stderr:\n{output}",
709 output = output.strip()
710 if not output:
713 output="",
717 broken_files = [x.strip() for x in output.splitlines()]
718 autofix = [gofmt, "-w"] + broken_files
721 output="The following Go files have incorrect "
730 files: List[str],
731 ) -> CheckResult:
735 if not any(x.startswith(compiler_wrapper_prefix) for x in files):
738 output="no compiler_wrapper changes detected",
745 output=textwrap.dedent(
747 Compiler wrapper changes should be made in chromiumos-overlay.
761 files: List[str],
762 ) -> CheckResult:
765 [os.path.join(toolchain_utils_root, "run_tests_for.py"), "--"] + files,
770 output=stdout_and_stderr,
775 def detect_toolchain_utils_root() -> str:
783 ) -> Tuple[bool, List[List[str]]]:
784 """Prints human-readable output for the given check_results."""
787 def indent_block(text: str) -> str:
791 ok, output, autofix_commands = check_results
797 if output:
798 output += "\n" + recommendation
800 output = recommendation
804 for subname, (ok, output, autofix) in check_results:
807 if output:
808 message.append(indent_block(output))
821 output = "\n\n".join(output_pieces)
823 time_taken = datetime.datetime.now() - start_time
829 if output:
830 print(indent_block(output))
838 ) -> None:
843 exit_code, output = run_command_unchecked(
844 ["git", "status", "--porcelain"], cwd=toolchain_utils_root
847 print("Autofix aborted: couldn't get toolchain-utils git status.")
850 if output.strip():
858 exit_code, output = run_command_unchecked(
867 print(output)
879 def find_repo_root(base_dir: str) -> Optional[str]:
888 def is_in_chroot() -> bool:
893 autofix: bool, install_deps_only: bool, files: List[str]
894 ) -> None:
905 "Standalone toolchain-utils checkout detected; cannot enter "
923 # We'll be changing ${PWD}, so make everything relative to toolchain-utils,
924 # which resides at a well-known place inside of the chroot.
925 chroot_toolchain_utils = "/mnt/host/source/src/third_party/toolchain-utils"
927 def rebase_path(path: str) -> str:
934 "--enter",
935 "--",
940 args.append("--no_autofix")
942 args.append("--install_deps_only")
943 args.extend(rebase_path(x) for x in files)
953 def can_import_py_module(module: str) -> bool:
956 ["python3", "-c", f"import {module}"],
963 def ensure_pip_deps_installed() -> None:
972 subprocess.check_call(pip + ["install", "--user", package])
975 def main(argv: List[str]) -> int:
978 "--no_autofix",
984 "--no_enter_chroot",
987 help="Prevent auto-entering the chroot if we're not already in it.",
990 "--install_deps_only",
997 parser.add_argument("files", nargs="*")
1000 files = opts.files
1002 if not files and not install_deps_only:
1006 maybe_reexec_inside_chroot(opts.autofix, install_deps_only, files)
1008 # If you ask for --no_enter_chroot, you're on your own for installing these
1014 "Dependency installation complete & --install_deps_only "
1020 "--install_deps_only is meaningless if the chroot isn't entered"
1023 files = [os.path.abspath(f) for f in files]
1026 # user-friendly way.
1051 return name, check_fn(toolchain_utils_root, pool, files)
1064 # - we don't collide with checkers that are running concurrently
1065 # - we clearly print out everything that went wrong ahead of time, in case