1# Copyright © 2019-2020 Intel Corporation 2 3# Permission is hereby granted, free of charge, to any person obtaining a copy 4# of this software and associated documentation files (the "Software"), to deal 5# in the Software without restriction, including without limitation the rights 6# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7# copies of the Software, and to permit persons to whom the Software is 8# furnished to do so, subject to the following conditions: 9 10# The above copyright notice and this permission notice shall be included in 11# all copies or substantial portions of the Software. 12 13# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19# SOFTWARE. 20 21"""Core data structures and routines for pick.""" 22 23import asyncio 24import enum 25import json 26import pathlib 27import re 28import subprocess 29import typing 30 31import attr 32 33if typing.TYPE_CHECKING: 34 from .ui import UI 35 36 import typing_extensions 37 38 class CommitDict(typing_extensions.TypedDict): 39 40 sha: str 41 description: str 42 nominated: bool 43 nomination_type: typing.Optional[int] 44 resolution: typing.Optional[int] 45 main_sha: typing.Optional[str] 46 because_sha: typing.Optional[str] 47 48IS_FIX = re.compile(r'^\s*fixes:\s*([a-f0-9]{6,40})', flags=re.MULTILINE | re.IGNORECASE) 49# FIXME: I dislike the duplication in this regex, but I couldn't get it to work otherwise 50IS_CC = re.compile(r'^\s*cc:\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*\<?mesa-stable', 51 flags=re.MULTILINE | re.IGNORECASE) 52IS_REVERT = re.compile(r'This reverts commit ([0-9a-f]{40})') 53 54# XXX: hack 55SEM = asyncio.Semaphore(50) 56 57COMMIT_LOCK = asyncio.Lock() 58 59git_toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], 60 stderr=subprocess.DEVNULL).decode("ascii").strip() 61pick_status_json = pathlib.Path(git_toplevel) / '.pick_status.json' 62 63 64class PickUIException(Exception): 65 pass 66 67 68@enum.unique 69class NominationType(enum.Enum): 70 71 CC = 0 72 FIXES = 1 73 REVERT = 2 74 75 76@enum.unique 77class Resolution(enum.Enum): 78 79 UNRESOLVED = 0 80 MERGED = 1 81 DENOMINATED = 2 82 BACKPORTED = 3 83 NOTNEEDED = 4 84 85 86async def commit_state(*, amend: bool = False, message: str = 'Update') -> bool: 87 """Commit the .pick_status.json file.""" 88 async with COMMIT_LOCK: 89 p = await asyncio.create_subprocess_exec( 90 'git', 'add', pick_status_json.as_posix(), 91 stdout=asyncio.subprocess.DEVNULL, 92 stderr=asyncio.subprocess.DEVNULL, 93 ) 94 v = await p.wait() 95 if v != 0: 96 return False 97 98 if amend: 99 cmd = ['--amend', '--no-edit'] 100 else: 101 cmd = ['--message', f'.pick_status.json: {message}'] 102 p = await asyncio.create_subprocess_exec( 103 'git', 'commit', *cmd, 104 stdout=asyncio.subprocess.DEVNULL, 105 stderr=asyncio.subprocess.DEVNULL, 106 ) 107 v = await p.wait() 108 if v != 0: 109 return False 110 return True 111 112 113@attr.s(slots=True) 114class Commit: 115 116 sha: str = attr.ib() 117 description: str = attr.ib() 118 nominated: bool = attr.ib(False) 119 nomination_type: typing.Optional[NominationType] = attr.ib(None) 120 resolution: Resolution = attr.ib(Resolution.UNRESOLVED) 121 main_sha: typing.Optional[str] = attr.ib(None) 122 because_sha: typing.Optional[str] = attr.ib(None) 123 124 def to_json(self) -> 'CommitDict': 125 d: typing.Dict[str, typing.Any] = attr.asdict(self) 126 if self.nomination_type is not None: 127 d['nomination_type'] = self.nomination_type.value 128 if self.resolution is not None: 129 d['resolution'] = self.resolution.value 130 return typing.cast('CommitDict', d) 131 132 @classmethod 133 def from_json(cls, data: 'CommitDict') -> 'Commit': 134 c = cls(data['sha'], data['description'], data['nominated'], main_sha=data['main_sha'], because_sha=data['because_sha']) 135 if data['nomination_type'] is not None: 136 c.nomination_type = NominationType(data['nomination_type']) 137 if data['resolution'] is not None: 138 c.resolution = Resolution(data['resolution']) 139 return c 140 141 def date(self) -> str: 142 # Show commit date, ie. when the commit actually landed 143 # (as opposed to when it was first written) 144 return subprocess.check_output( 145 ['git', 'show', '--no-patch', '--format=%cs', self.sha], 146 stderr=subprocess.DEVNULL 147 ).decode("ascii").strip() 148 149 async def apply(self, ui: 'UI') -> typing.Tuple[bool, str]: 150 # FIXME: This isn't really enough if we fail to cherry-pick because the 151 # git tree will still be dirty 152 async with COMMIT_LOCK: 153 p = await asyncio.create_subprocess_exec( 154 'git', 'cherry-pick', '-x', self.sha, 155 stdout=asyncio.subprocess.DEVNULL, 156 stderr=asyncio.subprocess.PIPE, 157 ) 158 _, err = await p.communicate() 159 160 if p.returncode != 0: 161 return (False, err.decode()) 162 163 self.resolution = Resolution.MERGED 164 await ui.feedback(f'{self.sha} ({self.description}) applied successfully') 165 166 # Append the changes to the .pickstatus.json file 167 ui.save() 168 v = await commit_state(amend=True) 169 return (v, '') 170 171 async def abort_cherry(self, ui: 'UI', err: str) -> None: 172 await ui.feedback(f'{self.sha} ({self.description}) failed to apply\n{err}') 173 async with COMMIT_LOCK: 174 p = await asyncio.create_subprocess_exec( 175 'git', 'cherry-pick', '--abort', 176 stdout=asyncio.subprocess.DEVNULL, 177 stderr=asyncio.subprocess.DEVNULL, 178 ) 179 r = await p.wait() 180 await ui.feedback(f'{"Successfully" if r == 0 else "Failed to"} abort cherry-pick.') 181 182 async def denominate(self, ui: 'UI') -> bool: 183 self.resolution = Resolution.DENOMINATED 184 ui.save() 185 v = await commit_state(message=f'Mark {self.sha} as denominated') 186 assert v 187 await ui.feedback(f'{self.sha} ({self.description}) denominated successfully') 188 return True 189 190 async def backport(self, ui: 'UI') -> bool: 191 self.resolution = Resolution.BACKPORTED 192 ui.save() 193 v = await commit_state(message=f'Mark {self.sha} as backported') 194 assert v 195 await ui.feedback(f'{self.sha} ({self.description}) backported successfully') 196 return True 197 198 async def resolve(self, ui: 'UI') -> None: 199 self.resolution = Resolution.MERGED 200 ui.save() 201 v = await commit_state(amend=True) 202 assert v 203 await ui.feedback(f'{self.sha} ({self.description}) committed successfully') 204 205 206async def get_new_commits(sha: str) -> typing.List[typing.Tuple[str, str]]: 207 # Try to get the authoritative upstream main 208 p = await asyncio.create_subprocess_exec( 209 'git', 'for-each-ref', '--format=%(upstream)', 'refs/heads/main', 210 stdout=asyncio.subprocess.PIPE, 211 stderr=asyncio.subprocess.DEVNULL) 212 out, _ = await p.communicate() 213 upstream = out.decode().strip() 214 215 p = await asyncio.create_subprocess_exec( 216 'git', 'log', '--pretty=oneline', f'{sha}..{upstream}', 217 stdout=asyncio.subprocess.PIPE, 218 stderr=asyncio.subprocess.DEVNULL) 219 out, _ = await p.communicate() 220 assert p.returncode == 0, f"git log didn't work: {sha}" 221 return list(split_commit_list(out.decode().strip())) 222 223 224def split_commit_list(commits: str) -> typing.Generator[typing.Tuple[str, str], None, None]: 225 if not commits: 226 return 227 for line in commits.split('\n'): 228 v = tuple(line.split(' ', 1)) 229 assert len(v) == 2, 'this is really just for mypy' 230 yield typing.cast(typing.Tuple[str, str], v) 231 232 233async def is_commit_in_branch(sha: str) -> bool: 234 async with SEM: 235 p = await asyncio.create_subprocess_exec( 236 'git', 'merge-base', '--is-ancestor', sha, 'HEAD', 237 stdout=asyncio.subprocess.DEVNULL, 238 stderr=asyncio.subprocess.DEVNULL, 239 ) 240 await p.wait() 241 return p.returncode == 0 242 243 244async def full_sha(sha: str) -> str: 245 async with SEM: 246 p = await asyncio.create_subprocess_exec( 247 'git', 'rev-parse', sha, 248 stdout=asyncio.subprocess.PIPE, 249 stderr=asyncio.subprocess.DEVNULL, 250 ) 251 out, _ = await p.communicate() 252 if p.returncode: 253 raise PickUIException(f'Invalid Sha {sha}') 254 return out.decode().strip() 255 256 257async def resolve_nomination(commit: 'Commit', version: str) -> 'Commit': 258 async with SEM: 259 p = await asyncio.create_subprocess_exec( 260 'git', 'log', '--format=%B', '-1', commit.sha, 261 stdout=asyncio.subprocess.PIPE, 262 stderr=asyncio.subprocess.DEVNULL, 263 ) 264 _out, _ = await p.communicate() 265 assert p.returncode == 0, f'git log for {commit.sha} failed' 266 out = _out.decode() 267 268 # We give precedence to fixes and cc tags over revert tags. 269 # XXX: not having the walrus operator available makes me sad := 270 m = IS_FIX.search(out) 271 if m: 272 # We set the nomination_type and because_sha here so that we can later 273 # check to see if this fixes another staged commit. 274 try: 275 commit.because_sha = fixed = await full_sha(m.group(1)) 276 except PickUIException: 277 pass 278 else: 279 commit.nomination_type = NominationType.FIXES 280 if await is_commit_in_branch(fixed): 281 commit.nominated = True 282 return commit 283 284 m = IS_CC.search(out) 285 if m: 286 if m.groups() == (None, None) or version in m.groups(): 287 commit.nominated = True 288 commit.nomination_type = NominationType.CC 289 return commit 290 291 m = IS_REVERT.search(out) 292 if m: 293 # See comment for IS_FIX path 294 try: 295 commit.because_sha = reverted = await full_sha(m.group(1)) 296 except PickUIException: 297 pass 298 else: 299 commit.nomination_type = NominationType.REVERT 300 if await is_commit_in_branch(reverted): 301 commit.nominated = True 302 return commit 303 304 return commit 305 306 307async def resolve_fixes(commits: typing.List['Commit'], previous: typing.List['Commit']) -> None: 308 """Determine if any of the undecided commits fix/revert a staged commit. 309 310 The are still needed if they apply to a commit that is staged for 311 inclusion, but not yet included. 312 313 This must be done in order, because a commit 3 might fix commit 2 which 314 fixes commit 1. 315 """ 316 shas: typing.Set[str] = set(c.sha for c in previous if c.nominated) 317 assert None not in shas, 'None in shas' 318 319 for commit in reversed(commits): 320 if not commit.nominated and commit.nomination_type is NominationType.FIXES: 321 commit.nominated = commit.because_sha in shas 322 323 if commit.nominated: 324 shas.add(commit.sha) 325 326 for commit in commits: 327 if (commit.nomination_type is NominationType.REVERT and 328 commit.because_sha in shas): 329 for oldc in reversed(commits): 330 if oldc.sha == commit.because_sha: 331 # In this case a commit that hasn't yet been applied is 332 # reverted, we don't want to apply that commit at all 333 oldc.nominated = False 334 oldc.resolution = Resolution.DENOMINATED 335 commit.nominated = False 336 commit.resolution = Resolution.DENOMINATED 337 shas.remove(commit.because_sha) 338 break 339 340 341async def gather_commits(version: str, previous: typing.List['Commit'], 342 new: typing.List[typing.Tuple[str, str]], cb) -> typing.List['Commit']: 343 # We create an array of the final size up front, then we pass that array 344 # to the "inner" co-routine, which is turned into a list of tasks and 345 # collected by asyncio.gather. We do this to allow the tasks to be 346 # asynchronously gathered, but to also ensure that the commits list remains 347 # in order. 348 m_commits: typing.List[typing.Optional['Commit']] = [None] * len(new) 349 tasks = [] 350 351 async def inner(commit: 'Commit', version: str, 352 commits: typing.List[typing.Optional['Commit']], 353 index: int, cb) -> None: 354 commits[index] = await resolve_nomination(commit, version) 355 cb() 356 357 for i, (sha, desc) in enumerate(new): 358 tasks.append(asyncio.ensure_future( 359 inner(Commit(sha, desc), version, m_commits, i, cb))) 360 361 await asyncio.gather(*tasks) 362 assert None not in m_commits 363 commits = typing.cast(typing.List[Commit], m_commits) 364 365 await resolve_fixes(commits, previous) 366 367 for commit in commits: 368 if commit.resolution is Resolution.UNRESOLVED and not commit.nominated: 369 commit.resolution = Resolution.NOTNEEDED 370 371 return commits 372 373 374def load() -> typing.List['Commit']: 375 if not pick_status_json.exists(): 376 return [] 377 with pick_status_json.open('r') as f: 378 raw = json.load(f) 379 return [Commit.from_json(c) for c in raw] 380 381 382def save(commits: typing.Iterable['Commit']) -> None: 383 commits = list(commits) 384 with pick_status_json.open('wt') as f: 385 json.dump([c.to_json() for c in commits], f, indent=4) 386 387 asyncio.ensure_future(commit_state(message=f'Update to {commits[0].sha}')) 388