• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2019 gRPC authors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Generate draft and release notes in Markdown from Github PRs.
15
16You'll need a github API token to avoid being rate-limited. See
17https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
18
19This script collects PRs using "git log X..Y" from local repo where X and Y are
20tags or release branch names of previous and current releases respectively.
21Typically, notes are generated before the release branch is labelled so Y is
22almost always the name of the release branch. X is the previous release branch
23if this is not a patch release. Otherwise, it is the previous release tag.
24For example, for release v1.17.0, X will be origin/v1.16.x and for release v1.17.3,
25X will be v1.17.2. In both cases Y will be origin/v1.17.x.
26
27"""
28
29from collections import defaultdict
30import json
31import logging
32import re
33import subprocess
34
35import urllib3
36
37logging.basicConfig(level=logging.WARNING)
38
39content_header = """Draft Release Notes For {version}
40--
41Final release notes will be generated from the PR titles that have *"release notes:yes"* label. If you have any additional notes please add them below. These will be appended to auto generated release notes. Previous release notes are [here](https://github.com/grpc/grpc/releases).
42
43**Also, look at the PRs listed below against your name.** Please apply the missing labels and make necessary corrections (like fixing the title) to the PR in Github. Final release notes will be generated just before the release on {date}.
44
45Add additional notes not in PRs
46--
47
48Core
49-
50
51
52C++
53-
54
55
56C#
57-
58
59
60Objective-C
61-
62
63
64PHP
65-
66
67
68Python
69-
70
71
72Ruby
73-
74
75
76"""
77
78rl_header = """This is release {version} ([{name}](https://github.com/grpc/grpc/blob/master/doc/g_stands_for.md)) of gRPC Core.
79
80For gRPC documentation, see [grpc.io](https://grpc.io/). For previous releases, see [Releases](https://github.com/grpc/grpc/releases).
81
82This release contains refinements, improvements, and bug fixes, with highlights listed below.
83
84
85"""
86
87HTML_URL = "https://github.com/grpc/grpc/pull/"
88API_URL = "https://api.github.com/repos/grpc/grpc/pulls/"
89
90
91def get_commit_detail(commit):
92    """Print commit and CL info for the commits that are submitted with CL-first workflow and warn the release manager to check manually."""
93    glg_command = [
94        "git",
95        "log",
96        "-n 1",
97        "%s" % commit,
98    ]
99    output = subprocess.check_output(glg_command).decode("utf-8", "ignore")
100    matches = re.search("Author:.*<(.*@).*>", output)
101    author = matches.group(1)
102    detail = "- " + author + " "
103    title = output.splitlines()[4].strip()
104    detail += "- " + title
105    if not title.endswith("."):
106        detail += "."
107    detail += " ([commit](https://github.com/grpc/grpc/commit/{}))".format(
108        commit
109    )
110    matches = re.search("PiperOrigin-RevId: ([0-9]+)$", output)
111    # backport commits might not have PiperOrigin-RevId
112    if matches is not None:
113        cl_num = matches.group(1)
114        detail += " ([CL](https://critique.corp.google.com/cl/{}))".format(
115            cl_num
116        )
117    return detail
118
119
120def get_commit_log(prevRelLabel, relBranch):
121    """Return the output of 'git log prevRelLabel..relBranch'"""
122
123    import subprocess
124
125    glg_command = [
126        "git",
127        "log",
128        "--pretty=oneline",
129        "%s..%s" % (prevRelLabel, relBranch),
130    ]
131    print(("Running ", " ".join(glg_command)))
132    return subprocess.check_output(glg_command).decode("utf-8", "ignore")
133
134
135def get_pr_data(pr_num):
136    """Get the PR data from github. Return 'error' on exception"""
137    http = urllib3.PoolManager(
138        retries=urllib3.Retry(total=7, backoff_factor=1), timeout=4.0
139    )
140    url = API_URL + pr_num
141    try:
142        response = http.request(
143            "GET", url, headers={"Authorization": "token %s" % TOKEN}
144        )
145    except urllib3.exceptions.HTTPError as e:
146        print("Request error:", e.reason)
147        return "error"
148    return json.loads(response.data.decode("utf-8"))
149
150
151def get_pr_titles(gitLogs):
152    import re
153
154    # All commits
155    match_commit = "^([a-fA-F0-9]+) "
156    all_commits_set = set(re.findall(match_commit, gitLogs, re.MULTILINE))
157
158    error_count = 0
159    # PRs with merge commits
160    match_merge_pr = "^([a-fA-F0-9]+) .*Merge pull request #(\d+)"
161    matches = re.findall(match_merge_pr, gitLogs, re.MULTILINE)
162    merge_commits = []
163    prlist_merge_pr = []
164    if matches:
165        merge_commits, prlist_merge_pr = zip(*matches)
166    merge_commits_set = set(merge_commits)
167    print("\nPRs matching 'Merge pull request #<num>':")
168    print(prlist_merge_pr)
169    print("\n")
170
171    # PRs using Github's squash & merge feature
172    match_sq = "^([a-fA-F0-9]+) .*\(#(\d+)\)$"
173    matches = re.findall(match_sq, gitLogs, re.MULTILINE)
174    if matches:
175        sq_commits, prlist_sq = zip(*matches)
176    sq_commits_set = set(sq_commits)
177    print("\nPRs matching '[PR Description](#<num>)$'")
178    print(prlist_sq)
179    print("\n")
180    prlist = list(prlist_merge_pr) + list(prlist_sq)
181    langs_pr = defaultdict(list)
182    for pr_num in prlist:
183        pr_num = str(pr_num)
184        print(("---------- getting data for PR " + pr_num))
185        pr = get_pr_data(pr_num)
186        if pr == "error":
187            print(
188                ("\n***ERROR*** Error in getting data for PR " + pr_num + "\n")
189            )
190            error_count += 1
191            continue
192        rl_no_found = False
193        rl_yes_found = False
194        lang_found = False
195        for label in pr["labels"]:
196            if label["name"] == "release notes: yes":
197                rl_yes_found = True
198            elif label["name"] == "release notes: no":
199                rl_no_found = True
200            elif label["name"].startswith("lang/"):
201                lang_found = True
202                lang = label["name"].split("/")[1].lower()
203                # lang = lang[0].upper() + lang[1:]
204        body = pr["title"]
205        if not body.endswith("."):
206            body = body + "."
207
208        prline = (
209            "-  " + body + " ([#" + pr_num + "](" + HTML_URL + pr_num + "))"
210        )
211        detail = "- " + pr["user"]["login"] + "@ " + prline
212        print(detail)
213        # if no RL label
214        if not rl_no_found and not rl_yes_found:
215            print(("Release notes label missing for " + pr_num))
216            langs_pr["nolabel"].append(detail)
217        elif rl_yes_found and not lang_found:
218            print(("Lang label missing for " + pr_num))
219            langs_pr["nolang"].append(detail)
220        elif rl_no_found:
221            print(("'Release notes:no' found for " + pr_num))
222            langs_pr["notinrel"].append(detail)
223        elif rl_yes_found:
224            print(
225                (
226                    "'Release notes:yes' found for "
227                    + pr_num
228                    + " with lang "
229                    + lang
230                )
231            )
232            langs_pr["inrel"].append(detail)
233            langs_pr[lang].append(prline)
234    commits_wo_pr = all_commits_set - merge_commits_set - sq_commits_set
235    for commit in commits_wo_pr:
236        langs_pr["nopr"].append(get_commit_detail(commit))
237
238    return langs_pr, error_count
239
240
241def write_draft(langs_pr, file, version, date):
242    file.write(content_header.format(version=version, date=date))
243    file.write(
244        "Commits with missing PR number - please lookup the PR info in the corresponding CL and add to the additional notes if necessary.\n"
245    )
246    file.write("---\n")
247    file.write("\n")
248    if langs_pr["nopr"]:
249        file.write("\n".join(langs_pr["nopr"]))
250    else:
251        file.write("- None")
252    file.write("\n")
253    file.write("\n")
254    file.write("PRs with missing release notes label - please fix in Github\n")
255    file.write("---\n")
256    file.write("\n")
257    if langs_pr["nolabel"]:
258        langs_pr["nolabel"].sort()
259        file.write("\n".join(langs_pr["nolabel"]))
260    else:
261        file.write("- None")
262    file.write("\n")
263    file.write("\n")
264    file.write("PRs with missing lang label - please fix in Github\n")
265    file.write("---\n")
266    file.write("\n")
267    if langs_pr["nolang"]:
268        langs_pr["nolang"].sort()
269        file.write("\n".join(langs_pr["nolang"]))
270    else:
271        file.write("- None")
272    file.write("\n")
273    file.write("\n")
274    file.write(
275        "PRs going into release notes - please check title and fix in Github."
276        " Do not edit here.\n"
277    )
278    file.write("---\n")
279    file.write("\n")
280    if langs_pr["inrel"]:
281        langs_pr["inrel"].sort()
282        file.write("\n".join(langs_pr["inrel"]))
283    else:
284        file.write("- None")
285    file.write("\n")
286    file.write("\n")
287    file.write("PRs not going into release notes\n")
288    file.write("---\n")
289    file.write("\n")
290    if langs_pr["notinrel"]:
291        langs_pr["notinrel"].sort()
292        file.write("\n".join(langs_pr["notinrel"]))
293    else:
294        file.write("- None")
295    file.write("\n")
296    file.write("\n")
297
298
299def write_rel_notes(langs_pr, file, version, name):
300    file.write(rl_header.format(version=version, name=name))
301    if langs_pr["core"]:
302        file.write("Core\n---\n\n")
303        file.write("\n".join(langs_pr["core"]))
304        file.write("\n")
305        file.write("\n")
306    if langs_pr["c++"]:
307        file.write("C++\n---\n\n")
308        file.write("\n".join(langs_pr["c++"]))
309        file.write("\n")
310        file.write("\n")
311    if langs_pr["c#"]:
312        file.write("C#\n---\n\n")
313        file.write("\n".join(langs_pr["c#"]))
314        file.write("\n")
315        file.write("\n")
316    if langs_pr["go"]:
317        file.write("Go\n---\n\n")
318        file.write("\n".join(langs_pr["go"]))
319        file.write("\n")
320        file.write("\n")
321    if langs_pr["Java"]:
322        file.write("Java\n---\n\n")
323        file.write("\n".join(langs_pr["Java"]))
324        file.write("\n")
325        file.write("\n")
326    if langs_pr["node"]:
327        file.write("Node\n---\n\n")
328        file.write("\n".join(langs_pr["node"]))
329        file.write("\n")
330        file.write("\n")
331    if langs_pr["objc"]:
332        file.write("Objective-C\n---\n\n")
333        file.write("\n".join(langs_pr["objc"]))
334        file.write("\n")
335        file.write("\n")
336    if langs_pr["php"]:
337        file.write("PHP\n---\n\n")
338        file.write("\n".join(langs_pr["php"]))
339        file.write("\n")
340        file.write("\n")
341    if langs_pr["python"]:
342        file.write("Python\n---\n\n")
343        file.write("\n".join(langs_pr["python"]))
344        file.write("\n")
345        file.write("\n")
346    if langs_pr["ruby"]:
347        file.write("Ruby\n---\n\n")
348        file.write("\n".join(langs_pr["ruby"]))
349        file.write("\n")
350        file.write("\n")
351    if langs_pr["other"]:
352        file.write("Other\n---\n\n")
353        file.write("\n".join(langs_pr["other"]))
354        file.write("\n")
355        file.write("\n")
356
357
358def build_args_parser():
359    import argparse
360
361    parser = argparse.ArgumentParser()
362    parser.add_argument(
363        "release_version", type=str, help="New release version e.g. 1.14.0"
364    )
365    parser.add_argument(
366        "release_name", type=str, help="New release name e.g. gladiolus"
367    )
368    parser.add_argument(
369        "release_date", type=str, help="Release date e.g. 7/30/18"
370    )
371    parser.add_argument(
372        "previous_release_label",
373        type=str,
374        help="Previous release branch/tag e.g. v1.13.x",
375    )
376    parser.add_argument(
377        "release_branch",
378        type=str,
379        help="Current release branch e.g. origin/v1.14.x",
380    )
381    parser.add_argument(
382        "draft_filename", type=str, help="Name of the draft file e.g. draft.md"
383    )
384    parser.add_argument(
385        "release_notes_filename",
386        type=str,
387        help="Name of the release notes file e.g. relnotes.md",
388    )
389    parser.add_argument(
390        "--token",
391        type=str,
392        default="",
393        help="GitHub API token to avoid being rate limited",
394    )
395    return parser
396
397
398def main():
399    import os
400
401    global TOKEN
402
403    parser = build_args_parser()
404    args = parser.parse_args()
405    version, name, date = (
406        args.release_version,
407        args.release_name,
408        args.release_date,
409    )
410    start, end = args.previous_release_label, args.release_branch
411
412    TOKEN = args.token
413    if TOKEN == "":
414        try:
415            TOKEN = os.environ["GITHUB_TOKEN"]
416        except:
417            pass
418    if TOKEN == "":
419        print(
420            "Error: Github API token required. Either include param"
421            " --token=<your github token> or set environment variable"
422            " GITHUB_TOKEN to your github token"
423        )
424        return
425
426    langs_pr, error_count = get_pr_titles(get_commit_log(start, end))
427
428    draft_file, rel_file = args.draft_filename, args.release_notes_filename
429    filename = os.path.abspath(draft_file)
430    if os.path.exists(filename):
431        file = open(filename, "r+")
432    else:
433        file = open(filename, "w")
434
435    file.seek(0)
436    write_draft(langs_pr, file, version, date)
437    file.truncate()
438    file.close()
439    print(("\nDraft notes written to " + filename))
440
441    filename = os.path.abspath(rel_file)
442    if os.path.exists(filename):
443        file = open(filename, "r+")
444    else:
445        file = open(filename, "w")
446
447    file.seek(0)
448    write_rel_notes(langs_pr, file, version, name)
449    file.truncate()
450    file.close()
451    print(("\nRelease notes written to " + filename))
452    if error_count > 0:
453        print("\n\n*** Errors were encountered. See log. *********\n")
454
455
456if __name__ == "__main__":
457    main()
458