• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright 2020 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Checks for new reverts in LLVM on a nightly basis.
8
9If any reverts are found that were previously unknown, this cherry-picks them or
10fires off an email. All LLVM SHAs to monitor are autodetected.
11"""
12
13from __future__ import print_function
14
15import argparse
16import io
17import json
18import logging
19import os
20import pprint
21import subprocess
22import sys
23import typing as t
24
25import cros_utils.email_sender as email_sender
26import cros_utils.tiny_render as tiny_render
27
28import get_llvm_hash
29import get_upstream_patch
30import git_llvm_rev
31import revert_checker
32
33State = t.Any
34
35
36def _find_interesting_android_shas(android_llvm_toolchain_dir: str
37                                   ) -> t.List[t.Tuple[str, str]]:
38  llvm_project = os.path.join(android_llvm_toolchain_dir,
39                              'toolchain/llvm-project')
40
41  def get_llvm_merge_base(branch: str) -> str:
42    head_sha = subprocess.check_output(
43        ['git', 'rev-parse', branch],
44        cwd=llvm_project,
45        encoding='utf-8',
46    ).strip()
47    merge_base = subprocess.check_output(
48        ['git', 'merge-base', branch, 'aosp/upstream-main'],
49        cwd=llvm_project,
50        encoding='utf-8',
51    ).strip()
52    logging.info('Merge-base for %s (HEAD == %s) and upstream-main is %s',
53                 branch, head_sha, merge_base)
54    return merge_base
55
56  main_legacy = get_llvm_merge_base('aosp/master-legacy')  # nocheck
57  testing_upstream = get_llvm_merge_base('aosp/testing-upstream')
58  result = [('main-legacy', main_legacy)]
59
60  # If these are the same SHA, there's no point in tracking both.
61  if main_legacy != testing_upstream:
62    result.append(('testing-upstream', testing_upstream))
63  else:
64    logging.info('main-legacy and testing-upstream are identical; ignoring '
65                 'the latter.')
66  return result
67
68
69def _parse_llvm_ebuild_for_shas(ebuild_file: io.TextIOWrapper
70                                ) -> t.List[t.Tuple[str, str]]:
71  def parse_ebuild_assignment(line: str) -> str:
72    no_comments = line.split('#')[0]
73    no_assign = no_comments.split('=', 1)[1].strip()
74    assert no_assign.startswith('"') and no_assign.endswith('"'), no_assign
75    return no_assign[1:-1]
76
77  llvm_hash, llvm_next_hash = None, None
78  for line in ebuild_file:
79    if line.startswith('LLVM_HASH='):
80      llvm_hash = parse_ebuild_assignment(line)
81      if llvm_next_hash:
82        break
83    if line.startswith('LLVM_NEXT_HASH'):
84      llvm_next_hash = parse_ebuild_assignment(line)
85      if llvm_hash:
86        break
87  if not llvm_next_hash or not llvm_hash:
88    raise ValueError('Failed to detect SHAs for llvm/llvm_next. Got: '
89                     'llvm=%s; llvm_next=%s' % (llvm_hash, llvm_next_hash))
90
91  results = [('llvm', llvm_hash)]
92  if llvm_next_hash != llvm_hash:
93    results.append(('llvm-next', llvm_next_hash))
94  return results
95
96
97def _find_interesting_chromeos_shas(chromeos_base: str
98                                    ) -> t.List[t.Tuple[str, str]]:
99  llvm_dir = os.path.join(chromeos_base,
100                          'src/third_party/chromiumos-overlay/sys-devel/llvm')
101  candidate_ebuilds = [
102      os.path.join(llvm_dir, x) for x in os.listdir(llvm_dir)
103      if '_pre' in x and not os.path.islink(os.path.join(llvm_dir, x))
104  ]
105
106  if len(candidate_ebuilds) != 1:
107    raise ValueError('Expected exactly one llvm ebuild candidate; got %s' %
108                     pprint.pformat(candidate_ebuilds))
109
110  with open(candidate_ebuilds[0], encoding='utf-8') as f:
111    return _parse_llvm_ebuild_for_shas(f)
112
113
114_Email = t.NamedTuple('_Email', [
115    ('subject', str),
116    ('body', tiny_render.Piece),
117])
118
119
120def _generate_revert_email(
121    repository_name: str, friendly_name: str, sha: str,
122    prettify_sha: t.Callable[[str], tiny_render.Piece],
123    get_sha_description: t.Callable[[str], tiny_render.Piece],
124    new_reverts: t.List[revert_checker.Revert]) -> _Email:
125  email_pieces = [
126      'It looks like there may be %s across %s (' % (
127          'a new revert' if len(new_reverts) == 1 else 'new reverts',
128          friendly_name,
129      ),
130      prettify_sha(sha),
131      ').',
132      tiny_render.line_break,
133      tiny_render.line_break,
134      'That is:' if len(new_reverts) == 1 else 'These are:',
135  ]
136
137  revert_listing = []
138  for revert in sorted(new_reverts, key=lambda r: r.sha):
139    revert_listing.append([
140        prettify_sha(revert.sha),
141        ' (appears to revert ',
142        prettify_sha(revert.reverted_sha),
143        '): ',
144        get_sha_description(revert.sha),
145    ])
146
147  email_pieces.append(tiny_render.UnorderedList(items=revert_listing))
148  email_pieces += [
149      tiny_render.line_break,
150      'PTAL and consider reverting them locally.',
151  ]
152  return _Email(
153      subject='[revert-checker/%s] new %s discovered across %s' % (
154          repository_name,
155          'revert' if len(new_reverts) == 1 else 'reverts',
156          friendly_name,
157      ),
158      body=email_pieces,
159  )
160
161
162_EmailRecipients = t.NamedTuple(
163    '_EmailRecipients',
164    [
165        ('well_known', t.List[str]),
166        ('direct', t.List[str]),
167    ],
168)
169
170
171def _send_revert_email(recipients: _EmailRecipients, email: _Email) -> None:
172  email_sender.EmailSender().SendX20Email(
173      subject=email.subject,
174      identifier='revert-checker',
175      well_known_recipients=recipients.well_known,
176      direct_recipients=['gbiv@google.com'] + recipients.direct,
177      text_body=tiny_render.render_text_pieces(email.body),
178      html_body=tiny_render.render_html_pieces(email.body),
179  )
180
181
182def _write_state(state_file: str, new_state: State) -> None:
183  try:
184    tmp_file = state_file + '.new'
185    with open(tmp_file, 'w', encoding='utf-8') as f:
186      json.dump(new_state, f, sort_keys=True, indent=2, separators=(',', ': '))
187    os.rename(tmp_file, state_file)
188  except:
189    try:
190      os.remove(tmp_file)
191    except FileNotFoundError:
192      pass
193    raise
194
195
196def _read_state(state_file: str) -> State:
197  try:
198    with open(state_file) as f:
199      return json.load(f)
200  except FileNotFoundError:
201    logging.info('No state file found at %r; starting with an empty slate',
202                 state_file)
203    return {}
204
205
206def find_shas(llvm_dir: str, interesting_shas: t.List[t.Tuple[str, str]],
207              state: State, new_state: State):
208  for friendly_name, sha in interesting_shas:
209    logging.info('Finding reverts across %s (%s)', friendly_name, sha)
210    all_reverts = revert_checker.find_reverts(llvm_dir,
211                                              sha,
212                                              root='origin/' +
213                                              git_llvm_rev.MAIN_BRANCH)
214    logging.info('Detected the following revert(s) across %s:\n%s',
215                 friendly_name, pprint.pformat(all_reverts))
216
217    new_state[sha] = [r.sha for r in all_reverts]
218
219    if sha not in state:
220      logging.info('SHA %s is new to me', sha)
221      existing_reverts = set()
222    else:
223      existing_reverts = set(state[sha])
224
225    new_reverts = [r for r in all_reverts if r.sha not in existing_reverts]
226    if not new_reverts:
227      logging.info('...All of which have been reported.')
228      continue
229
230    yield (friendly_name, sha, new_reverts)
231
232
233def do_cherrypick(chroot_path: str, llvm_dir: str,
234                  interesting_shas: t.List[t.Tuple[str, str]], state: State,
235                  reviewers: t.List[str], cc: t.List[str]) -> State:
236  new_state: State = {}
237  seen: t.Set[str] = set()
238  for friendly_name, _sha, reverts in find_shas(llvm_dir, interesting_shas,
239                                                state, new_state):
240    if friendly_name in seen:
241      continue
242    seen.add(friendly_name)
243    for sha, reverted_sha in reverts:
244      try:
245        # We upload reverts for all platforms by default, since there's no
246        # real reason for them to be CrOS-specific.
247        get_upstream_patch.get_from_upstream(chroot_path=chroot_path,
248                                             create_cl=True,
249                                             start_sha=reverted_sha,
250                                             patches=[sha],
251                                             reviewers=reviewers,
252                                             cc=cc,
253                                             platforms=())
254      except get_upstream_patch.CherrypickError as e:
255        logging.info('%s, skipping...', str(e))
256  return new_state
257
258
259def do_email(is_dry_run: bool, llvm_dir: str, repository: str,
260             interesting_shas: t.List[t.Tuple[str, str]], state: State,
261             recipients: _EmailRecipients) -> State:
262  def prettify_sha(sha: str) -> tiny_render.Piece:
263    rev = get_llvm_hash.GetVersionFrom(llvm_dir, sha)
264
265    # 12 is arbitrary, but should be unambiguous enough.
266    short_sha = sha[:12]
267    return tiny_render.Switch(
268        text=f'r{rev} ({short_sha})',
269        html=tiny_render.Link(href='https://reviews.llvm.org/rG' + sha,
270                              inner='r' + str(rev)),
271    )
272
273  def get_sha_description(sha: str) -> tiny_render.Piece:
274    return subprocess.check_output(
275        ['git', 'log', '-n1', '--format=%s', sha],
276        cwd=llvm_dir,
277        encoding='utf-8',
278    ).strip()
279
280  new_state: State = {}
281  for friendly_name, sha, new_reverts in find_shas(llvm_dir, interesting_shas,
282                                                   state, new_state):
283    email = _generate_revert_email(repository, friendly_name, sha,
284                                   prettify_sha, get_sha_description,
285                                   new_reverts)
286    if is_dry_run:
287      logging.info('Would send email:\nSubject: %s\nBody:\n%s\n',
288                   email.subject, tiny_render.render_text_pieces(email.body))
289    else:
290      logging.info('Sending email with subject %r...', email.subject)
291      _send_revert_email(recipients, email)
292      logging.info('Email sent.')
293  return new_state
294
295
296def parse_args(argv: t.List[str]) -> t.Any:
297  parser = argparse.ArgumentParser(
298      description=__doc__,
299      formatter_class=argparse.RawDescriptionHelpFormatter)
300  parser.add_argument(
301      'action',
302      choices=['cherry-pick', 'email', 'dry-run'],
303      help='Automatically cherry-pick upstream reverts, send an email, or '
304      'write to stdout.')
305  parser.add_argument('--state_file',
306                      required=True,
307                      help='File to store persistent state in.')
308  parser.add_argument('--llvm_dir',
309                      required=True,
310                      help='Up-to-date LLVM directory to use.')
311  parser.add_argument('--debug', action='store_true')
312  parser.add_argument(
313      '--reviewers',
314      type=str,
315      nargs='*',
316      help='Requests reviews from REVIEWERS. All REVIEWERS must have existing '
317      'accounts.')
318  parser.add_argument(
319      '--cc',
320      type=str,
321      nargs='*',
322      help='CCs the CL to the recipients. All recipients must have existing '
323      'accounts.')
324
325  subparsers = parser.add_subparsers(dest='repository')
326  subparsers.required = True
327
328  chromeos_subparser = subparsers.add_parser('chromeos')
329  chromeos_subparser.add_argument('--chromeos_dir',
330                                  required=True,
331                                  help='Up-to-date CrOS directory to use.')
332
333  android_subparser = subparsers.add_parser('android')
334  android_subparser.add_argument(
335      '--android_llvm_toolchain_dir',
336      required=True,
337      help='Up-to-date android-llvm-toolchain directory to use.')
338
339  return parser.parse_args(argv)
340
341
342def find_chroot(opts: t.Any, reviewers: t.List[str], cc: t.List[str]
343                ) -> t.Tuple[str, t.List[t.Tuple[str, str]], _EmailRecipients]:
344  recipients = reviewers + cc
345  if opts.repository == 'chromeos':
346    chroot_path = opts.chromeos_dir
347    return (chroot_path, _find_interesting_chromeos_shas(chroot_path),
348            _EmailRecipients(well_known=['mage'], direct=recipients))
349  elif opts.repository == 'android':
350    if opts.action == 'cherry-pick':
351      raise RuntimeError(
352          "android doesn't currently support automatic cherry-picking.")
353
354    chroot_path = opts.android_llvm_toolchain_dir
355    return (chroot_path, _find_interesting_android_shas(chroot_path),
356            _EmailRecipients(well_known=[],
357                             direct=['android-llvm-dev@google.com'] +
358                             recipients))
359  else:
360    raise ValueError(f'Unknown repository {opts.repository}')
361
362
363def main(argv: t.List[str]) -> int:
364  opts = parse_args(argv)
365
366  logging.basicConfig(
367      format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s',
368      level=logging.DEBUG if opts.debug else logging.INFO,
369  )
370
371  action = opts.action
372  llvm_dir = opts.llvm_dir
373  repository = opts.repository
374  state_file = opts.state_file
375  reviewers = opts.reviewers if opts.reviewers else []
376  cc = opts.cc if opts.cc else []
377
378  chroot_path, interesting_shas, recipients = find_chroot(opts, reviewers, cc)
379  logging.info('Interesting SHAs were %r', interesting_shas)
380
381  state = _read_state(state_file)
382  logging.info('Loaded state\n%s', pprint.pformat(state))
383
384  # We want to be as free of obvious side-effects as possible in case something
385  # above breaks. Hence, action as late as possible.
386  if action == 'cherry-pick':
387    new_state = do_cherrypick(chroot_path=chroot_path,
388                              llvm_dir=llvm_dir,
389                              interesting_shas=interesting_shas,
390                              state=state,
391                              reviewers=reviewers,
392                              cc=cc)
393  else:
394    new_state = do_email(is_dry_run=action == 'dry-run',
395                         llvm_dir=llvm_dir,
396                         repository=repository,
397                         interesting_shas=interesting_shas,
398                         state=state,
399                         recipients=recipients)
400
401  _write_state(state_file, new_state)
402  return 0
403
404
405if __name__ == '__main__':
406  sys.exit(main(sys.argv[1:]))
407