• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright 2020 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Get an upstream patch to LLVM's PATCHES.json."""
8
9import argparse
10import json
11import logging
12import os
13import shlex
14import subprocess
15import sys
16import typing as t
17from datetime import datetime
18
19import dataclasses
20
21import chroot
22import get_llvm_hash
23import git
24import git_llvm_rev
25import update_chromeos_llvm_hash
26
27__DOC_EPILOGUE = """
28Example Usage:
29  get_upstream_patch --chroot_path ~/chromiumos --platform chromiumos \
30--sha 1234567 --sha 890abdc
31"""
32
33
34class CherrypickError(ValueError):
35  """A ValueError that highlights the cherry-pick has been seen before"""
36
37
38def add_patch(patches_json_path: str, patches_dir: str,
39              relative_patches_dir: str, start_version: git_llvm_rev.Rev,
40              llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, str], sha: str,
41              package: str, platforms: t.List[str]):
42  """Gets the start and end intervals in 'json_file'.
43
44  Args:
45    patches_json_path: The absolute path to PATCHES.json.
46    patches_dir: The aboslute path to the directory patches are in.
47    relative_patches_dir: The relative path to PATCHES.json.
48    start_version: The base LLVM revision this patch applies to.
49    llvm_dir: The path to LLVM checkout.
50    rev: An LLVM revision (git_llvm_rev.Rev) for a cherrypicking, or a
51    differential revision (str) otherwise.
52    sha: The LLVM git sha that corresponds to the patch. For differential
53    revisions, the git sha from  the local commit created by 'arc patch'
54    is used.
55    package: The LLVM project name this patch applies to.
56    platforms: List of platforms this patch applies to.
57
58  Raises:
59    CherrypickError: A ValueError that highlights the cherry-pick has been
60    seen before.
61  """
62
63  with open(patches_json_path, encoding='utf-8') as f:
64    patches_json = json.load(f)
65
66  is_cherrypick = isinstance(rev, git_llvm_rev.Rev)
67  if is_cherrypick:
68    file_name = f'{sha}.patch'
69  else:
70    file_name = f'{rev}.patch'
71  rel_patch_path = os.path.join(relative_patches_dir, file_name)
72
73  for p in patches_json:
74    rel_path = p['rel_patch_path']
75    if rel_path == rel_patch_path:
76      raise CherrypickError(
77          f'Patch at {rel_path} already exists in PATCHES.json')
78    if is_cherrypick:
79      if sha in rel_path:
80        logging.warning(
81            'Similarly-named patch already exists in PATCHES.json: %r',
82            rel_path)
83
84  with open(os.path.join(patches_dir, file_name), 'wb') as f:
85    cmd = ['git', 'show', sha]
86    # Only apply the part of the patch that belongs to this package, expect
87    # LLVM. This is because some packages are built with LLVM ebuild on X86 but
88    # not on the other architectures. e.g. compiler-rt. Therefore always apply
89    # the entire patch to LLVM ebuild as a workaround.
90    if package != 'llvm':
91      cmd.append(package_to_project(package))
92    subprocess.check_call(cmd, stdout=f, cwd=llvm_dir)
93
94  commit_subject = subprocess.check_output(
95      ['git', 'log', '-n1', '--format=%s', sha],
96      cwd=llvm_dir,
97      encoding='utf-8')
98
99  end_vers = rev.number if isinstance(rev, git_llvm_rev.Rev) else None
100  patch_props = {
101      'rel_patch_path': rel_patch_path,
102      'metadata': {
103          'title': commit_subject.strip(),
104          'info': [],
105      },
106      'platforms': sorted(platforms),
107      'version_range': {
108          'from': start_version.number,
109          'until': end_vers,
110      },
111  }
112  patches_json.append(patch_props)
113
114  temp_file = patches_json_path + '.tmp'
115  with open(temp_file, 'w', encoding='utf-8') as f:
116    json.dump(patches_json,
117              f,
118              indent=4,
119              separators=(',', ': '),
120              sort_keys=True)
121    f.write('\n')
122  os.rename(temp_file, patches_json_path)
123
124
125def parse_ebuild_for_assignment(ebuild_path: str, var_name: str) -> str:
126  # '_pre' filters the LLVM 9.0 ebuild, which we never want to target, from
127  # this list.
128  candidates = [
129      x for x in os.listdir(ebuild_path)
130      if x.endswith('.ebuild') and '_pre' in x
131  ]
132
133  if not candidates:
134    raise ValueError('No ebuilds found under %r' % ebuild_path)
135
136  ebuild = os.path.join(ebuild_path, max(candidates))
137  with open(ebuild, encoding='utf-8') as f:
138    var_name_eq = var_name + '='
139    for orig_line in f:
140      if not orig_line.startswith(var_name_eq):
141        continue
142
143      # We shouldn't see much variety here, so do the simplest thing possible.
144      line = orig_line[len(var_name_eq):]
145      # Remove comments
146      line = line.split('#')[0]
147      # Remove quotes
148      line = shlex.split(line)
149      if len(line) != 1:
150        raise ValueError('Expected exactly one quoted value in %r' % orig_line)
151      return line[0].strip()
152
153  raise ValueError('No %s= line found in %r' % (var_name, ebuild))
154
155
156# Resolves a git ref (or similar) to a LLVM SHA.
157def resolve_llvm_ref(llvm_dir: str, sha: str) -> str:
158  return subprocess.check_output(
159      ['git', 'rev-parse', sha],
160      encoding='utf-8',
161      cwd=llvm_dir,
162  ).strip()
163
164
165# Get the package name of an LLVM project
166def project_to_package(project: str) -> str:
167  if project == 'libunwind':
168    return 'llvm-libunwind'
169  return project
170
171
172# Get the LLVM project name of a package
173def package_to_project(package: str) -> str:
174  if package == 'llvm-libunwind':
175    return 'libunwind'
176  return package
177
178
179# Get the LLVM projects change in the specifed sha
180def get_package_names(sha: str, llvm_dir: str) -> list:
181  paths = subprocess.check_output(
182      ['git', 'show', '--name-only', '--format=', sha],
183      cwd=llvm_dir,
184      encoding='utf-8').splitlines()
185  # Some LLVM projects are built by LLVM ebuild on X86, so always apply the
186  # patch to LLVM ebuild
187  packages = {'llvm'}
188  # Detect if there are more packages to apply the patch to
189  for path in paths:
190    package = project_to_package(path.split('/')[0])
191    if package in ('compiler-rt', 'libcxx', 'libcxxabi', 'llvm-libunwind'):
192      packages.add(package)
193  packages = list(sorted(packages))
194  return packages
195
196
197def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str],
198                              start_rev: git_llvm_rev.Rev,
199                              rev: t.Union[git_llvm_rev.Rev, str], sha: str,
200                              llvm_dir: str, platforms: t.List[str]):
201  """Create a patch and add its metadata for each package"""
202  for package, symlink in zip(packages, symlinks):
203    symlink_dir = os.path.dirname(symlink)
204    patches_json_path = os.path.join(symlink_dir, 'files/PATCHES.json')
205    relative_patches_dir = 'cherry' if package == 'llvm' else ''
206    patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir)
207    logging.info('Getting %s (%s) into %s', rev, sha, package)
208    add_patch(patches_json_path,
209              patches_dir,
210              relative_patches_dir,
211              start_rev,
212              llvm_dir,
213              rev,
214              sha,
215              package,
216              platforms=platforms)
217
218
219def make_cl(symlinks_to_uprev: t.List[str], llvm_symlink_dir: str, branch: str,
220            commit_messages: t.List[str], reviewers: t.Optional[t.List[str]],
221            cc: t.Optional[t.List[str]]):
222  symlinks_to_uprev = sorted(set(symlinks_to_uprev))
223  for symlink in symlinks_to_uprev:
224    update_chromeos_llvm_hash.UprevEbuildSymlink(symlink)
225    subprocess.check_output(['git', 'add', '--all'],
226                            cwd=os.path.dirname(symlink))
227  git.UploadChanges(llvm_symlink_dir, branch, commit_messages, reviewers, cc)
228  git.DeleteBranch(llvm_symlink_dir, branch)
229
230
231def resolve_symbolic_sha(start_sha: str, llvm_symlink_dir: str) -> str:
232  if start_sha == 'llvm':
233    return parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_HASH')
234
235  if start_sha == 'llvm-next':
236    return parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_NEXT_HASH')
237
238  return start_sha
239
240
241def find_patches_and_make_cl(
242    chroot_path: str, patches: t.List[str], start_rev: git_llvm_rev.Rev,
243    llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str,
244    create_cl: bool, skip_dependencies: bool,
245    reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]],
246    platforms: t.List[str]):
247
248  converted_patches = [
249      _convert_patch(llvm_config, skip_dependencies, p) for p in patches
250  ]
251  potential_duplicates = _get_duplicate_shas(converted_patches)
252  if potential_duplicates:
253    err_msg = '\n'.join(f'{a.patch} == {b.patch}'
254                        for a, b in potential_duplicates)
255    raise RuntimeError(f'Found Duplicate SHAs:\n{err_msg}')
256
257  # CL Related variables, only used if `create_cl`
258  symlinks_to_uprev = []
259  commit_messages = [
260      'llvm: get patches from upstream\n',
261  ]
262  branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}'
263
264  if create_cl:
265    git.CreateBranch(llvm_symlink_dir, branch)
266
267  for parsed_patch in converted_patches:
268    # Find out the llvm projects changed in this commit
269    packages = get_package_names(parsed_patch.sha, llvm_config.dir)
270    # Find out the ebuild symlinks of the corresponding ChromeOS packages
271    symlinks = chroot.GetChrootEbuildPaths(chroot_path, [
272        'sys-devel/llvm' if package == 'llvm' else 'sys-libs/' + package
273        for package in packages
274    ])
275    symlinks = chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, symlinks)
276    # Create a local patch for all the affected llvm projects
277    create_patch_for_packages(packages,
278                              symlinks,
279                              start_rev,
280                              parsed_patch.rev,
281                              parsed_patch.sha,
282                              llvm_config.dir,
283                              platforms=platforms)
284    if create_cl:
285      symlinks_to_uprev.extend(symlinks)
286
287      commit_messages.extend([
288          parsed_patch.git_msg(),
289          subprocess.check_output(
290              ['git', 'log', '-n1', '--oneline', parsed_patch.sha],
291              cwd=llvm_config.dir,
292              encoding='utf-8')
293      ])
294
295    if parsed_patch.is_differential:
296      subprocess.check_output(['git', 'reset', '--hard', 'HEAD^'],
297                              cwd=llvm_config.dir)
298
299  if create_cl:
300    make_cl(symlinks_to_uprev, llvm_symlink_dir, branch, commit_messages,
301            reviewers, cc)
302
303
304@dataclasses.dataclass(frozen=True)
305class ParsedPatch:
306  """Class to keep track of bundled patch info."""
307  patch: str
308  sha: str
309  is_differential: bool
310  rev: t.Union[git_llvm_rev.Rev, str]
311
312  def git_msg(self) -> str:
313    if self.is_differential:
314      return f'\n\nreviews.llvm.org/{self.patch}\n'
315    return f'\n\nreviews.llvm.org/rG{self.sha}\n'
316
317
318def _convert_patch(llvm_config: git_llvm_rev.LLVMConfig,
319                   skip_dependencies: bool, patch: str) -> ParsedPatch:
320  """Extract git revision info from a patch.
321
322  Args:
323    llvm_config: LLVM configuration object.
324    skip_dependencies: Pass --skip-dependecies for to `arc`
325    patch: A single patch referent string.
326
327  Returns:
328    A [ParsedPatch] object.
329  """
330
331  # git hash should only have lower-case letters
332  is_differential = patch.startswith('D')
333  if is_differential:
334    subprocess.check_output(
335        [
336            'arc', 'patch', '--nobranch',
337            '--skip-dependencies' if skip_dependencies else '--revision', patch
338        ],
339        cwd=llvm_config.dir,
340    )
341    sha = resolve_llvm_ref(llvm_config.dir, 'HEAD')
342    rev = patch
343  else:
344    sha = resolve_llvm_ref(llvm_config.dir, patch)
345    rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha)
346  return ParsedPatch(patch=patch,
347                     sha=sha,
348                     rev=rev,
349                     is_differential=is_differential)
350
351
352def _get_duplicate_shas(patches: t.List[ParsedPatch]
353                        ) -> t.List[t.Tuple[ParsedPatch, ParsedPatch]]:
354  """Return a list of Patches which have duplicate SHA's"""
355  return [(left, right) for i, left in enumerate(patches)
356          for right in patches[i + 1:] if left.sha == right.sha]
357
358
359def get_from_upstream(chroot_path: str,
360                      create_cl: bool,
361                      start_sha: str,
362                      patches: t.List[str],
363                      platforms: t.List[str],
364                      skip_dependencies: bool = False,
365                      reviewers: t.List[str] = None,
366                      cc: t.List[str] = None):
367  llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths(
368      chroot_path, chroot.GetChrootEbuildPaths(chroot_path,
369                                               ['sys-devel/llvm']))[0]
370  llvm_symlink_dir = os.path.dirname(llvm_symlink)
371
372  git_status = subprocess.check_output(['git', 'status', '-s'],
373                                       cwd=llvm_symlink_dir,
374                                       encoding='utf-8')
375
376  if git_status:
377    error_path = os.path.dirname(os.path.dirname(llvm_symlink_dir))
378    raise ValueError(f'Uncommited changes detected in {error_path}')
379
380  start_sha = resolve_symbolic_sha(start_sha, llvm_symlink_dir)
381  logging.info('Base llvm hash == %s', start_sha)
382
383  llvm_config = git_llvm_rev.LLVMConfig(
384      remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools())
385  start_sha = resolve_llvm_ref(llvm_config.dir, start_sha)
386
387  find_patches_and_make_cl(chroot_path=chroot_path,
388                           patches=patches,
389                           platforms=platforms,
390                           start_rev=git_llvm_rev.translate_sha_to_rev(
391                               llvm_config, start_sha),
392                           llvm_config=llvm_config,
393                           llvm_symlink_dir=llvm_symlink_dir,
394                           create_cl=create_cl,
395                           skip_dependencies=skip_dependencies,
396                           reviewers=reviewers,
397                           cc=cc)
398  logging.info('Complete.')
399
400
401def main():
402  chroot.VerifyOutsideChroot()
403  logging.basicConfig(
404      format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s',
405      level=logging.INFO,
406  )
407
408  parser = argparse.ArgumentParser(
409      description=__doc__,
410      formatter_class=argparse.RawDescriptionHelpFormatter,
411      epilog=__DOC_EPILOGUE)
412  parser.add_argument('--chroot_path',
413                      default=os.path.join(os.path.expanduser('~'),
414                                           'chromiumos'),
415                      help='the path to the chroot (default: %(default)s)')
416  parser.add_argument(
417      '--start_sha',
418      default='llvm-next',
419      help='LLVM SHA that the patch should start applying at. You can specify '
420      '"llvm" or "llvm-next", as well. Defaults to %(default)s.')
421  parser.add_argument('--sha',
422                      action='append',
423                      default=[],
424                      help='The LLVM git SHA to cherry-pick.')
425  parser.add_argument(
426      '--differential',
427      action='append',
428      default=[],
429      help='The LLVM differential revision to apply. Example: D1234')
430  parser.add_argument(
431      '--platform',
432      action='append',
433      required=True,
434      help='Apply this patch to the give platform. Common options include '
435      '"chromiumos" and "android". Can be specified multiple times to '
436      'apply to multiple platforms')
437  parser.add_argument('--create_cl',
438                      action='store_true',
439                      help='Automatically create a CL if specified')
440  parser.add_argument(
441      '--skip_dependencies',
442      action='store_true',
443      help="Skips a LLVM differential revision's dependencies. Only valid "
444      'when --differential appears exactly once.')
445  args = parser.parse_args()
446
447  if not (args.sha or args.differential):
448    parser.error('--sha or --differential required')
449
450  if args.skip_dependencies and len(args.differential) != 1:
451    parser.error("--skip_dependencies is only valid when there's exactly one "
452                 'supplied differential')
453
454  get_from_upstream(
455      chroot_path=args.chroot_path,
456      create_cl=args.create_cl,
457      start_sha=args.start_sha,
458      patches=args.sha + args.differential,
459      skip_dependencies=args.skip_dependencies,
460      platforms=args.platform,
461  )
462
463
464if __name__ == '__main__':
465  sys.exit(main())
466