• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# This script computes the new "current" toolstate for the toolstate repo (not to be
5# confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
6# It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
7# (i.e., after it passed all checks on `auto`).
8
9from __future__ import print_function
10
11import sys
12import re
13import os
14import json
15import datetime
16import collections
17import textwrap
18try:
19    import urllib2
20    from urllib2 import HTTPError
21except ImportError:
22    import urllib.request as urllib2
23    from urllib.error import HTTPError
24try:
25    import typing # noqa: F401 FIXME: py2
26except ImportError:
27    pass
28
29# List of people to ping when the status of a tool or a book changed.
30# These should be collaborators of the rust-lang/rust repository (with at least
31# read privileges on it). CI will fail otherwise.
32MAINTAINERS = {
33    'book': {'carols10cents'},
34    'nomicon': {'frewsxcv', 'Gankra', 'JohnTitor'},
35    'reference': {'Havvy', 'matthewjasper', 'ehuss'},
36    'rust-by-example': {'marioidival'},
37    'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'},
38    'edition-guide': {'ehuss'},
39    'rustc-dev-guide': {'spastorino', 'amanjeev', 'JohnTitor'},
40}
41
42LABELS = {
43    'book': ['C-bug'],
44    'nomicon': ['C-bug'],
45    'reference': ['C-bug'],
46    'rust-by-example': ['C-bug'],
47    'embedded-book': ['C-bug'],
48    'edition-guide': ['C-bug'],
49    'rustc-dev-guide': ['C-bug'],
50}
51
52REPOS = {
53    'book': 'https://github.com/rust-lang/book',
54    'nomicon': 'https://github.com/rust-lang/nomicon',
55    'reference': 'https://github.com/rust-lang/reference',
56    'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
57    'embedded-book': 'https://github.com/rust-embedded/book',
58    'edition-guide': 'https://github.com/rust-lang/edition-guide',
59    'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
60}
61
62def load_json_from_response(resp):
63    # type: (typing.Any) -> typing.Any
64    content = resp.read()
65    if isinstance(content, bytes):
66        content_str = content.decode('utf-8')
67    else:
68        print("Refusing to decode " + str(type(content)) + " to str")
69    return json.loads(content_str)
70
71
72def read_current_status(current_commit, path):
73    # type: (str, str) -> typing.Mapping[str, typing.Any]
74    '''Reads build status of `current_commit` from content of `history/*.tsv`
75    '''
76    with open(path, 'r') as f:
77        for line in f:
78            (commit, status) = line.split('\t', 1)
79            if commit == current_commit:
80                return json.loads(status)
81    return {}
82
83
84def gh_url():
85    # type: () -> str
86    return os.environ['TOOLSTATE_ISSUES_API_URL']
87
88
89def maybe_remove_mention(message):
90    # type: (str) -> str
91    if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
92        return message.replace("@", "")
93    return message
94
95
96def issue(
97    tool,
98    status,
99    assignees,
100    relevant_pr_number,
101    relevant_pr_user,
102    labels,
103    github_token,
104):
105    # type: (str, str, typing.Iterable[str], str, str, typing.List[str], str) -> None
106    '''Open an issue about the toolstate failure.'''
107    if status == 'test-fail':
108        status_description = 'has failing tests'
109    else:
110        status_description = 'no longer builds'
111    request = json.dumps({
112        'body': maybe_remove_mention(textwrap.dedent('''\
113        Hello, this is your friendly neighborhood mergebot.
114        After merging PR {}, I observed that the tool {} {}.
115        A follow-up PR to the repository {} is needed to fix the fallout.
116
117        cc @{}, do you think you would have time to do the follow-up work?
118        If so, that would be great!
119        ''').format(
120            relevant_pr_number, tool, status_description,
121            REPOS.get(tool), relevant_pr_user
122        )),
123        'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
124        'assignees': list(assignees),
125        'labels': labels,
126    })
127    print("Creating issue:\n{}".format(request))
128    response = urllib2.urlopen(urllib2.Request(
129        gh_url(),
130        request.encode(),
131        {
132            'Authorization': 'token ' + github_token,
133            'Content-Type': 'application/json',
134        }
135    ))
136    response.read()
137
138
139def update_latest(
140    current_commit,
141    relevant_pr_number,
142    relevant_pr_url,
143    relevant_pr_user,
144    pr_reviewer,
145    current_datetime,
146    github_token,
147):
148    # type: (str, str, str, str, str, str, str) -> str
149    '''Updates `_data/latest.json` to match build result of the given commit.
150    '''
151    with open('_data/latest.json', 'r+') as f:
152        latest = json.load(f, object_pairs_hook=collections.OrderedDict)
153
154        current_status = {
155            os_: read_current_status(current_commit, 'history/' + os_ + '.tsv')
156            for os_ in ['windows', 'linux']
157        }
158
159        slug = 'rust-lang/rust'
160        message = textwrap.dedent('''\
161            �� Toolstate changed by {}!
162
163            Tested on commit {}@{}.
164            Direct link to PR: <{}>
165
166        ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
167        anything_changed = False
168        for status in latest:
169            tool = status['tool']
170            changed = False
171            create_issue_for_status = None  # set to the status that caused the issue
172
173            for os_, s in current_status.items():
174                old = status[os_]
175                new = s.get(tool, old)
176                status[os_] = new
177                maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
178                # comparing the strings, but they are ordered appropriately:
179                # "test-pass" > "test-fail" > "build-fail"
180                if new > old:
181                    # things got fixed or at least the status quo improved
182                    changed = True
183                    message += '�� {} on {}: {} → {} (cc {}).\n' \
184                        .format(tool, os_, old, new, maintainers)
185                elif new < old:
186                    # tests or builds are failing and were not failing before
187                    changed = True
188                    title = '�� {} on {}: {} → {}' \
189                        .format(tool, os_, old, new)
190                    message += '{} (cc {}).\n' \
191                        .format(title, maintainers)
192                    # See if we need to create an issue.
193                    # Create issue if things no longer build.
194                    # (No issue for mere test failures to avoid spurious issues.)
195                    if new == 'build-fail':
196                        create_issue_for_status = new
197
198            if create_issue_for_status is not None:
199                try:
200                    issue(
201                        tool, create_issue_for_status, MAINTAINERS.get(tool, ()),
202                        relevant_pr_number, relevant_pr_user, LABELS.get(tool, []),
203                        github_token,
204                    )
205                except HTTPError as e:
206                    # network errors will simply end up not creating an issue, but that's better
207                    # than failing the entire build job
208                    print("HTTPError when creating issue for status regression: {0}\n{1!r}"
209                          .format(e, e.read()))
210                except IOError as e:
211                    print("I/O error when creating issue for status regression: {0}".format(e))
212                except:
213                    print("Unexpected error when creating issue for status regression: {0}"
214                          .format(sys.exc_info()[0]))
215                    raise
216
217            if changed:
218                status['commit'] = current_commit
219                status['datetime'] = current_datetime
220                anything_changed = True
221
222        if not anything_changed:
223            return ''
224
225        f.seek(0)
226        f.truncate(0)
227        json.dump(latest, f, indent=4, separators=(',', ': '))
228        return message
229
230
231# Warning: Do not try to add a function containing the body of this try block.
232# There are variables declared within that are implicitly global; it is unknown
233# which ones precisely but at least this is true for `github_token`.
234try:
235    if __name__ != '__main__':
236        exit(0)
237
238    cur_commit = sys.argv[1]
239    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
240    cur_commit_msg = sys.argv[2]
241    save_message_to_path = sys.argv[3]
242    github_token = sys.argv[4]
243
244    # assume that PR authors are also owners of the repo where the branch lives
245    relevant_pr_match = re.search(
246        r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
247        cur_commit_msg,
248    )
249    if relevant_pr_match:
250        number = relevant_pr_match.group(1)
251        relevant_pr_user = relevant_pr_match.group(2)
252        relevant_pr_number = 'rust-lang/rust#' + number
253        relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
254        pr_reviewer = relevant_pr_match.group(3)
255    else:
256        number = '-1'
257        relevant_pr_user = 'ghost'
258        relevant_pr_number = '<unknown PR>'
259        relevant_pr_url = '<unknown>'
260        pr_reviewer = 'ghost'
261
262    message = update_latest(
263        cur_commit,
264        relevant_pr_number,
265        relevant_pr_url,
266        relevant_pr_user,
267        pr_reviewer,
268        cur_datetime,
269        github_token,
270    )
271    if not message:
272        print('<Nothing changed>')
273        sys.exit(0)
274
275    print(message)
276
277    if not github_token:
278        print('Dry run only, not committing anything')
279        sys.exit(0)
280
281    with open(save_message_to_path, 'w') as f:
282        f.write(message)
283
284    # Write the toolstate comment on the PR as well.
285    issue_url = gh_url() + '/{}/comments'.format(number)
286    response = urllib2.urlopen(urllib2.Request(
287        issue_url,
288        json.dumps({'body': maybe_remove_mention(message)}).encode(),
289        {
290            'Authorization': 'token ' + github_token,
291            'Content-Type': 'application/json',
292        }
293    ))
294    response.read()
295except HTTPError as e:
296    print("HTTPError: %s\n%r" % (e, e.read()))
297    raise
298