• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright 2020 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Checks for various upstream events with the Rust toolchain.
8
9Sends an email if something interesting (probably) happened.
10"""
11
12import argparse
13import itertools
14import json
15import logging
16import pathlib
17import re
18import shutil
19import subprocess
20import sys
21import time
22from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple
23
24from cros_utils import bugs, email_sender, tiny_render
25
26
27def gentoo_sha_to_link(sha: str) -> str:
28  """Gets a URL to a webpage that shows the Gentoo commit at `sha`."""
29  return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}'
30
31
32def send_email(subject: str, body: List[tiny_render.Piece]) -> None:
33  """Sends an email with the given title and body to... whoever cares."""
34  email_sender.EmailSender().SendX20Email(
35      subject=subject,
36      identifier='rust-watch',
37      well_known_recipients=['cros-team'],
38      text_body=tiny_render.render_text_pieces(body),
39      html_body=tiny_render.render_html_pieces(body),
40  )
41
42
43class RustReleaseVersion(NamedTuple):
44  """Represents a version of Rust's stable compiler."""
45  major: int
46  minor: int
47  patch: int
48
49  @staticmethod
50  def from_string(version_string: str) -> 'RustReleaseVersion':
51    m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string)
52    if not m:
53      raise ValueError(f"{version_string!r} isn't a valid version string")
54    return RustReleaseVersion(*[int(x) for x in m.groups()])
55
56  def __str__(self) -> str:
57    return f'{self.major}.{self.minor}.{self.patch}'
58
59  def to_json(self) -> str:
60    return str(self)
61
62  @staticmethod
63  def from_json(s: str) -> 'RustReleaseVersion':
64    return RustReleaseVersion.from_string(s)
65
66
67class State(NamedTuple):
68  """State that we keep around from run to run."""
69  # The last Rust release tag that we've seen.
70  last_seen_release: RustReleaseVersion
71
72  # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen
73  # that updates it.
74  last_gentoo_sha: str
75
76  def to_json(self) -> Dict[str, Any]:
77    return {
78        'last_seen_release': self.last_seen_release.to_json(),
79        'last_gentoo_sha': self.last_gentoo_sha,
80    }
81
82  @staticmethod
83  def from_json(s: Dict[str, Any]) -> 'State':
84    return State(
85        last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']),
86        last_gentoo_sha=s['last_gentoo_sha'],
87    )
88
89
90def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]:
91  """Parses `git ls-remote --tags` output into Rust stable release versions."""
92  refs_tags = 'refs/tags/'
93  for line in lines:
94    _sha, tag = line.split(None, 1)
95    tag = tag.strip()
96    # Each tag has an associated 'refs/tags/name^{}', which is the actual
97    # object that the tag points to. That's irrelevant to us.
98    if tag.endswith('^{}'):
99      continue
100
101    if not tag.startswith(refs_tags):
102      continue
103
104    short_tag = tag[len(refs_tags):]
105    # There are a few old versioning schemes. Ignore them.
106    if short_tag.startswith('0.') or short_tag.startswith('release-'):
107      continue
108    yield RustReleaseVersion.from_string(short_tag)
109
110
111def fetch_most_recent_release() -> RustReleaseVersion:
112  """Fetches the most recent stable `rustc` version."""
113  result = subprocess.run(
114      ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'],
115      check=True,
116      stdin=None,
117      capture_output=True,
118      encoding='utf-8',
119  )
120  tag_lines = result.stdout.strip().splitlines()
121  return max(parse_release_tags(tag_lines))
122
123
124class GitCommit(NamedTuple):
125  """Represents a single git commit."""
126  sha: str
127  subject: str
128
129
130def update_git_repo(git_dir: pathlib.Path) -> None:
131  """Updates the repo at `git_dir`, retrying a few times on failure."""
132  for i in itertools.count(start=1):
133    result = subprocess.run(
134        ['git', 'fetch', 'origin'],
135        check=False,
136        cwd=str(git_dir),
137        stdin=None,
138    )
139
140    if not result.returncode:
141      break
142
143    if i == 5:
144      # 5 attempts is too many. Something else may be wrong.
145      result.check_returncode()
146
147    sleep_time = 60 * i
148    logging.error("Failed updating gentoo's repo; will try again in %ds...",
149                  sleep_time)
150    time.sleep(sleep_time)
151
152
153def get_new_gentoo_commits(git_dir: pathlib.Path,
154                           most_recent_sha: str) -> List[GitCommit]:
155  """Gets commits to dev-lang/rust since `most_recent_sha`.
156
157  Older commits come earlier in the returned list.
158  """
159  commits = subprocess.run(
160      [
161          'git',
162          'log',
163          '--format=%H %s',
164          f'{most_recent_sha}..origin/master',  # nocheck
165          '--',
166          'dev-lang/rust',
167      ],
168      capture_output=True,
169      check=False,
170      cwd=str(git_dir),
171      encoding='utf-8',
172  )
173
174  if commits.returncode:
175    logging.error('Error getting new gentoo commits; stderr:\n%s',
176                  commits.stderr)
177    commits.check_returncode()
178
179  results = []
180  for line in commits.stdout.strip().splitlines():
181    sha, subject = line.strip().split(None, 1)
182    results.append(GitCommit(sha=sha, subject=subject))
183
184  # `git log` outputs things in newest -> oldest order.
185  results.reverse()
186  return results
187
188
189def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str:
190  """Sets up a gentoo git repo at the given directory. Returns HEAD."""
191  subprocess.run(
192      [
193          'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git',
194          str(git_dir)
195      ],
196      stdin=None,
197      check=True,
198  )
199
200  head_rev = subprocess.run(
201      ['git', 'rev-parse', 'HEAD'],
202      cwd=str(git_dir),
203      check=True,
204      stdin=None,
205      capture_output=True,
206      encoding='utf-8',
207  )
208  return head_rev.stdout.strip()
209
210
211def read_state(state_file: pathlib.Path) -> State:
212  """Reads state from the given file."""
213  with state_file.open(encoding='utf-8') as f:
214    return State.from_json(json.load(f))
215
216
217def atomically_write_state(state_file: pathlib.Path, state: State) -> None:
218  """Writes state to the given file."""
219  temp_file = pathlib.Path(str(state_file) + '.new')
220  with temp_file.open('w', encoding='utf-8') as f:
221    json.dump(state.to_json(), f)
222  temp_file.rename(state_file)
223
224
225def file_bug(title: str, body: str) -> None:
226  """Files a bug against gbiv@ with the given title/body."""
227  bugs.CreateNewBug(
228      bugs.WellKnownComponents.CrOSToolchainPublic,
229      title,
230      body,
231      # To either take or reassign depending on the rotation.
232      assignee='gbiv@google.com',
233  )
234
235
236def maybe_compose_bug(
237    old_state: State,
238    newest_release: RustReleaseVersion,
239) -> Optional[Tuple[str, str]]:
240  """Creates a bug to file about the new release, if doing is desired."""
241  if newest_release == old_state.last_seen_release:
242    return None
243
244  title = f'[Rust] Update to {newest_release}'
245  body = ('A new release has been detected; we should probably roll to it. '
246          "Please see go/crostc-rust-rotation for who's turn it is.")
247  return title, body
248
249
250def maybe_compose_email(
251    new_gentoo_commits: List[GitCommit]
252) -> Optional[Tuple[str, List[tiny_render.Piece]]]:
253  """Creates an email given our new state, if doing so is appropriate."""
254  if not new_gentoo_commits:
255    return None
256
257  subject_pieces = []
258  body_pieces = []
259
260  # Separate the sections a bit for prettier output.
261  if body_pieces:
262    body_pieces += [tiny_render.line_break, tiny_render.line_break]
263
264  if len(new_gentoo_commits) == 1:
265    subject_pieces.append('new rust ebuild commit detected')
266    body_pieces.append('commit:')
267  else:
268    subject_pieces.append('new rust ebuild commits detected')
269    body_pieces.append('commits (newest first):')
270
271  commit_lines = []
272  for commit in new_gentoo_commits:
273    commit_lines.append([
274        tiny_render.Link(
275            gentoo_sha_to_link(commit.sha),
276            commit.sha[:12],
277        ),
278        f': {commit.subject}',
279    ])
280
281  body_pieces.append(tiny_render.UnorderedList(commit_lines))
282
283  subject = '[rust-watch] ' + '; '.join(subject_pieces)
284  return subject, body_pieces
285
286
287def main(argv: List[str]) -> None:
288  logging.basicConfig(level=logging.INFO)
289
290  parser = argparse.ArgumentParser(
291      description=__doc__,
292      formatter_class=argparse.RawDescriptionHelpFormatter)
293  parser.add_argument('--state_dir',
294                      required=True,
295                      help='Directory to store state in.')
296  parser.add_argument('--skip_side_effects',
297                      action='store_true',
298                      help="Don't send an email or file a bug.")
299  parser.add_argument(
300      '--skip_state_update',
301      action='store_true',
302      help="Don't update the state file. Doesn't apply to initial setup.")
303  opts = parser.parse_args(argv)
304
305  state_dir = pathlib.Path(opts.state_dir)
306  state_file = state_dir / 'state.json'
307  gentoo_subdir = state_dir / 'upstream-gentoo'
308  if not state_file.exists():
309    logging.info("state_dir isn't fully set up; doing that now.")
310
311    # Could be in a partially set-up state.
312    if state_dir.exists():
313      logging.info('incomplete state_dir detected; removing.')
314      shutil.rmtree(str(state_dir))
315
316    state_dir.mkdir(parents=True)
317    most_recent_release = fetch_most_recent_release()
318    most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir)
319    atomically_write_state(
320        state_file,
321        State(
322            last_seen_release=most_recent_release,
323            last_gentoo_sha=most_recent_gentoo_commit,
324        ),
325    )
326    # Running through this _should_ be a nop, but do it anyway. Should make any
327    # bugs more obvious on the first run of the script.
328
329  prior_state = read_state(state_file)
330  logging.info('Last state was %r', prior_state)
331
332  most_recent_release = fetch_most_recent_release()
333  logging.info('Most recent Rust release is %s', most_recent_release)
334
335  logging.info('Fetching new commits from Gentoo')
336  update_git_repo(gentoo_subdir)
337  new_commits = get_new_gentoo_commits(gentoo_subdir,
338                                       prior_state.last_gentoo_sha)
339  logging.info('New commits: %r', new_commits)
340
341  maybe_bug = maybe_compose_bug(prior_state, most_recent_release)
342  maybe_email = maybe_compose_email(new_commits)
343
344  if maybe_bug is None:
345    logging.info('No bug to file')
346  else:
347    title, body = maybe_bug
348    if opts.skip_side_effects:
349      logging.info('Skipping sending bug with title %r and contents\n%s',
350                   title, body)
351    else:
352      logging.info('Writing new bug')
353      file_bug(title, body)
354
355  if maybe_email is None:
356    logging.info('No email to send')
357  else:
358    title, body = maybe_email
359    if opts.skip_side_effects:
360      logging.info('Skipping sending email with title %r and contents\n%s',
361                   title, tiny_render.render_html_pieces(body))
362    else:
363      logging.info('Sending email')
364      send_email(title, body)
365
366  if opts.skip_state_update:
367    logging.info('Skipping state update, as requested')
368    return
369
370  newest_sha = (new_commits[-1].sha
371                if new_commits else prior_state.last_gentoo_sha)
372  atomically_write_state(
373      state_file,
374      State(
375          last_seen_release=most_recent_release,
376          last_gentoo_sha=newest_sha,
377      ),
378  )
379
380
381if __name__ == '__main__':
382  sys.exit(main(sys.argv[1:]))
383