1# Copyright 2020 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""General purpose tools for running presubmit checks.""" 15 16import collections.abc 17from collections import Counter, defaultdict 18import logging 19import os 20from pathlib import Path 21import shlex 22import subprocess 23from typing import Any, Dict, Iterable, Iterator, List, Sequence, Pattern, Tuple 24 25_LOG: logging.Logger = logging.getLogger(__name__) 26 27 28def plural(items_or_count, 29 singular: str, 30 count_format='', 31 these: bool = False, 32 number: bool = True, 33 are: bool = False) -> str: 34 """Returns the singular or plural form of a word based on a count.""" 35 36 try: 37 count = len(items_or_count) 38 except TypeError: 39 count = items_or_count 40 41 prefix = ('this ' if count == 1 else 'these ') if these else '' 42 num = f'{count:{count_format}} ' if number else '' 43 suffix = (' is' if count == 1 else ' are') if are else '' 44 45 if singular.endswith('y'): 46 result = f'{singular[:-1]}{"y" if count == 1 else "ies"}' 47 elif singular.endswith('s'): 48 result = f'{singular}{"" if count == 1 else "es"}' 49 else: 50 result = f'{singular}{"" if count == 1 else "s"}' 51 52 return f'{prefix}{num}{result}{suffix}' 53 54 55def make_color(*codes: int): 56 start = ''.join(f'\033[{code}m' for code in codes) 57 return f'{start}{{}}\033[0m'.format if os.name == 'posix' else str 58 59 60def make_box(section_alignments: Sequence[str]) -> str: 61 indices = [i + 1 for i in range(len(section_alignments))] 62 top_sections = '{2}'.join('{1:{1}^{width%d}}' % i for i in indices) 63 mid_sections = '{5}'.join('{section%d:%s{width%d}}' % 64 (i, section_alignments[i - 1], i) 65 for i in indices) 66 bot_sections = '{9}'.join('{8:{8}^{width%d}}' % i for i in indices) 67 68 return ''.join(['{0}', *top_sections, '{3}\n', 69 '{4}', *mid_sections, '{6}\n', 70 '{7}', *bot_sections, '{10}']) # yapf: disable 71 72 73def file_summary(paths: Iterable[Path], 74 levels: int = 2, 75 max_lines: int = 12, 76 max_types: int = 3, 77 pad: str = ' ', 78 pad_start: str = ' ', 79 pad_end: str = ' ') -> List[str]: 80 """Summarizes a list of files by the file types in each directory.""" 81 82 # Count the file types in each directory. 83 all_counts: Dict[Any, Counter] = defaultdict(Counter) 84 85 for path in paths: 86 parent = path.parents[max(len(path.parents) - levels, 0)] 87 all_counts[parent][path.suffix] += 1 88 89 # If there are too many lines, condense directories with the fewest files. 90 if len(all_counts) > max_lines: 91 counts = sorted(all_counts.items(), 92 key=lambda item: -sum(item[1].values())) 93 counts, others = sorted(counts[:max_lines - 1]), counts[max_lines - 1:] 94 counts.append((f'({plural(others, "other")})', 95 sum((c for _, c in others), Counter()))) 96 else: 97 counts = sorted(all_counts.items()) 98 99 width = max(len(str(d)) + len(os.sep) for d, _ in counts) if counts else 0 100 width += len(pad_start) 101 102 # Prepare the output. 103 output = [] 104 for path, files in counts: 105 total = sum(files.values()) 106 del files[''] # Never display no-extension files individually. 107 108 if files: 109 extensions = files.most_common(max_types) 110 other_extensions = total - sum(count for _, count in extensions) 111 if other_extensions: 112 extensions.append(('other', other_extensions)) 113 114 types = ' (' + ', '.join(f'{c} {e}' for e, c in extensions) + ')' 115 else: 116 types = '' 117 118 root = f'{path}{os.sep}{pad_start}'.ljust(width, pad) 119 output.append(f'{root}{pad_end}{plural(total, "file")}{types}') 120 121 return output 122 123 124def relative_paths(paths: Iterable[Path], start: Path) -> Iterable[Path]: 125 """Returns relative Paths calculated with os.path.relpath.""" 126 for path in paths: 127 yield Path(os.path.relpath(path, start)) 128 129 130def exclude_paths(exclusions: Iterable[Pattern[str]], 131 paths: Iterable[Path], 132 relative_to: Path = None) -> Iterable[Path]: 133 """Excludes paths based on a series of regular expressions.""" 134 if relative_to: 135 relpath = lambda path: Path(os.path.relpath(path, relative_to)) 136 else: 137 relpath = lambda path: path 138 139 for path in paths: 140 if not any(e.search(relpath(path).as_posix()) for e in exclusions): 141 yield path 142 143 144def _truncate(value, length: int = 60) -> str: 145 value = str(value) 146 return (value[:length - 5] + '[...]') if len(value) > length else value 147 148 149def format_command(args: Sequence, kwargs: dict) -> Tuple[str, str]: 150 attr = ', '.join(f'{k}={_truncate(v)}' for k, v in sorted(kwargs.items())) 151 return attr, ' '.join(shlex.quote(str(arg)) for arg in args) 152 153 154def log_run(args, **kwargs) -> subprocess.CompletedProcess: 155 """Logs a command then runs it with subprocess.run. 156 157 Takes the same arguments as subprocess.run. 158 """ 159 _LOG.debug('[COMMAND] %s\n%s', *format_command(args, kwargs)) 160 return subprocess.run(args, **kwargs) 161 162 163def flatten(*items) -> Iterator: 164 """Yields items from a series of items and nested iterables. 165 166 This function is used to flatten arbitrarily nested lists. str and bytes 167 are kept intact. 168 """ 169 170 for item in items: 171 if isinstance(item, collections.abc.Iterable) and not isinstance( 172 item, (str, bytes, bytearray)): 173 yield from flatten(*item) 174 else: 175 yield item 176