• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2013 the V8 project authors. All rights reserved.
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8#       notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10#       copyright notice, this list of conditions and the following
11#       disclaimer in the documentation and/or other materials provided
12#       with the distribution.
13#     * Neither the name of Google Inc. nor the names of its
14#       contributors may be used to endorse or promote products derived
15#       from this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import argparse
30import datetime
31import imp
32import json
33import os
34import re
35import subprocess
36import sys
37import textwrap
38import time
39import urllib2
40
41from git_recipes import GitRecipesMixin
42from git_recipes import GitFailedException
43
44PERSISTFILE_BASENAME = "PERSISTFILE_BASENAME"
45BRANCHNAME = "BRANCHNAME"
46DOT_GIT_LOCATION = "DOT_GIT_LOCATION"
47VERSION_FILE = "VERSION_FILE"
48CHANGELOG_FILE = "CHANGELOG_FILE"
49CHANGELOG_ENTRY_FILE = "CHANGELOG_ENTRY_FILE"
50COMMITMSG_FILE = "COMMITMSG_FILE"
51PATCH_FILE = "PATCH_FILE"
52
53
54def TextToFile(text, file_name):
55  with open(file_name, "w") as f:
56    f.write(text)
57
58
59def AppendToFile(text, file_name):
60  with open(file_name, "a") as f:
61    f.write(text)
62
63
64def LinesInFile(file_name):
65  with open(file_name) as f:
66    for line in f:
67      yield line
68
69
70def FileToText(file_name):
71  with open(file_name) as f:
72    return f.read()
73
74
75def MSub(rexp, replacement, text):
76  return re.sub(rexp, replacement, text, flags=re.MULTILINE)
77
78
79def Fill80(line):
80  # Replace tabs and remove surrounding space.
81  line = re.sub(r"\t", r"        ", line.strip())
82
83  # Format with 8 characters indentation and line width 80.
84  return textwrap.fill(line, width=80, initial_indent="        ",
85                       subsequent_indent="        ")
86
87
88def MakeComment(text):
89  return MSub(r"^( ?)", "#", text)
90
91
92def StripComments(text):
93  # Use split not splitlines to keep terminal newlines.
94  return "\n".join(filter(lambda x: not x.startswith("#"), text.split("\n")))
95
96
97def MakeChangeLogBody(commit_messages, auto_format=False):
98  result = ""
99  added_titles = set()
100  for (title, body, author) in commit_messages:
101    # TODO(machenbach): Better check for reverts. A revert should remove the
102    # original CL from the actual log entry.
103    title = title.strip()
104    if auto_format:
105      # Only add commits that set the LOG flag correctly.
106      log_exp = r"^[ \t]*LOG[ \t]*=[ \t]*(?:(?:Y(?:ES)?)|TRUE)"
107      if not re.search(log_exp, body, flags=re.I | re.M):
108        continue
109      # Never include reverts.
110      if title.startswith("Revert "):
111        continue
112      # Don't include duplicates.
113      if title in added_titles:
114        continue
115
116    # Add and format the commit's title and bug reference. Move dot to the end.
117    added_titles.add(title)
118    raw_title = re.sub(r"(\.|\?|!)$", "", title)
119    bug_reference = MakeChangeLogBugReference(body)
120    space = " " if bug_reference else ""
121    result += "%s\n" % Fill80("%s%s%s." % (raw_title, space, bug_reference))
122
123    # Append the commit's author for reference if not in auto-format mode.
124    if not auto_format:
125      result += "%s\n" % Fill80("(%s)" % author.strip())
126
127    result += "\n"
128  return result
129
130
131def MakeChangeLogBugReference(body):
132  """Grep for "BUG=xxxx" lines in the commit message and convert them to
133  "(issue xxxx)".
134  """
135  crbugs = []
136  v8bugs = []
137
138  def AddIssues(text):
139    ref = re.match(r"^BUG[ \t]*=[ \t]*(.+)$", text.strip())
140    if not ref:
141      return
142    for bug in ref.group(1).split(","):
143      bug = bug.strip()
144      match = re.match(r"^v8:(\d+)$", bug)
145      if match: v8bugs.append(int(match.group(1)))
146      else:
147        match = re.match(r"^(?:chromium:)?(\d+)$", bug)
148        if match: crbugs.append(int(match.group(1)))
149
150  # Add issues to crbugs and v8bugs.
151  map(AddIssues, body.splitlines())
152
153  # Filter duplicates, sort, stringify.
154  crbugs = map(str, sorted(set(crbugs)))
155  v8bugs = map(str, sorted(set(v8bugs)))
156
157  bug_groups = []
158  def FormatIssues(prefix, bugs):
159    if len(bugs) > 0:
160      plural = "s" if len(bugs) > 1 else ""
161      bug_groups.append("%sissue%s %s" % (prefix, plural, ", ".join(bugs)))
162
163  FormatIssues("", v8bugs)
164  FormatIssues("Chromium ", crbugs)
165
166  if len(bug_groups) > 0:
167    return "(%s)" % ", ".join(bug_groups)
168  else:
169    return ""
170
171
172# Some commands don't like the pipe, e.g. calling vi from within the script or
173# from subscripts like git cl upload.
174def Command(cmd, args="", prefix="", pipe=True):
175  # TODO(machenbach): Use timeout.
176  cmd_line = "%s %s %s" % (prefix, cmd, args)
177  print "Command: %s" % cmd_line
178  sys.stdout.flush()
179  try:
180    if pipe:
181      return subprocess.check_output(cmd_line, shell=True)
182    else:
183      return subprocess.check_call(cmd_line, shell=True)
184  except subprocess.CalledProcessError:
185    return None
186  finally:
187    sys.stdout.flush()
188    sys.stderr.flush()
189
190
191# Wrapper for side effects.
192class SideEffectHandler(object):  # pragma: no cover
193  def Call(self, fun, *args, **kwargs):
194    return fun(*args, **kwargs)
195
196  def Command(self, cmd, args="", prefix="", pipe=True):
197    return Command(cmd, args, prefix, pipe)
198
199  def ReadLine(self):
200    return sys.stdin.readline().strip()
201
202  def ReadURL(self, url, params=None):
203    # pylint: disable=E1121
204    url_fh = urllib2.urlopen(url, params, 60)
205    try:
206      return url_fh.read()
207    finally:
208      url_fh.close()
209
210  def Sleep(self, seconds):
211    time.sleep(seconds)
212
213  def GetDate(self):
214    return datetime.date.today().strftime("%Y-%m-%d")
215
216DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler()
217
218
219class NoRetryException(Exception):
220  pass
221
222
223class Step(GitRecipesMixin):
224  def __init__(self, text, requires, number, config, state, options, handler):
225    self._text = text
226    self._requires = requires
227    self._number = number
228    self._config = config
229    self._state = state
230    self._options = options
231    self._side_effect_handler = handler
232    assert self._number >= 0
233    assert self._config is not None
234    assert self._state is not None
235    assert self._side_effect_handler is not None
236
237  def __getitem__(self, key):
238    # Convenience method to allow direct [] access on step classes for
239    # manipulating the backed state dict.
240    return self._state[key]
241
242  def __setitem__(self, key, value):
243    # Convenience method to allow direct [] access on step classes for
244    # manipulating the backed state dict.
245    self._state[key] = value
246
247  def Config(self, key):
248    return self._config[key]
249
250  def Run(self):
251    # Restore state.
252    state_file = "%s-state.json" % self._config[PERSISTFILE_BASENAME]
253    if not self._state and os.path.exists(state_file):
254      self._state.update(json.loads(FileToText(state_file)))
255
256    # Skip step if requirement is not met.
257    if self._requires and not self._state.get(self._requires):
258      return
259
260    print ">>> Step %d: %s" % (self._number, self._text)
261    try:
262      return self.RunStep()
263    finally:
264      # Persist state.
265      TextToFile(json.dumps(self._state), state_file)
266
267  def RunStep(self):  # pragma: no cover
268    raise NotImplementedError
269
270  def Retry(self, cb, retry_on=None, wait_plan=None):
271    """ Retry a function.
272    Params:
273      cb: The function to retry.
274      retry_on: A callback that takes the result of the function and returns
275                True if the function should be retried. A function throwing an
276                exception is always retried.
277      wait_plan: A list of waiting delays between retries in seconds. The
278                 maximum number of retries is len(wait_plan).
279    """
280    retry_on = retry_on or (lambda x: False)
281    wait_plan = list(wait_plan or [])
282    wait_plan.reverse()
283    while True:
284      got_exception = False
285      try:
286        result = cb()
287      except NoRetryException, e:
288        raise e
289      except Exception:
290        got_exception = True
291      if got_exception or retry_on(result):
292        if not wait_plan:  # pragma: no cover
293          raise Exception("Retried too often. Giving up.")
294        wait_time = wait_plan.pop()
295        print "Waiting for %f seconds." % wait_time
296        self._side_effect_handler.Sleep(wait_time)
297        print "Retrying..."
298      else:
299        return result
300
301  def ReadLine(self, default=None):
302    # Don't prompt in forced mode.
303    if self._options.force_readline_defaults and default is not None:
304      print "%s (forced)" % default
305      return default
306    else:
307      return self._side_effect_handler.ReadLine()
308
309  def Git(self, args="", prefix="", pipe=True, retry_on=None):
310    cmd = lambda: self._side_effect_handler.Command("git", args, prefix, pipe)
311    result = self.Retry(cmd, retry_on, [5, 30])
312    if result is None:
313      raise GitFailedException("'git %s' failed." % args)
314    return result
315
316  def SVN(self, args="", prefix="", pipe=True, retry_on=None):
317    cmd = lambda: self._side_effect_handler.Command("svn", args, prefix, pipe)
318    return self.Retry(cmd, retry_on, [5, 30])
319
320  def Editor(self, args):
321    if self._options.requires_editor:
322      return self._side_effect_handler.Command(os.environ["EDITOR"], args,
323                                               pipe=False)
324
325  def ReadURL(self, url, params=None, retry_on=None, wait_plan=None):
326    wait_plan = wait_plan or [3, 60, 600]
327    cmd = lambda: self._side_effect_handler.ReadURL(url, params)
328    return self.Retry(cmd, retry_on, wait_plan)
329
330  def GetDate(self):
331    return self._side_effect_handler.GetDate()
332
333  def Die(self, msg=""):
334    if msg != "":
335      print "Error: %s" % msg
336    print "Exiting"
337    raise Exception(msg)
338
339  def DieNoManualMode(self, msg=""):
340    if not self._options.manual:  # pragma: no cover
341      msg = msg or "Only available in manual mode."
342      self.Die(msg)
343
344  def Confirm(self, msg):
345    print "%s [Y/n] " % msg,
346    answer = self.ReadLine(default="Y")
347    return answer == "" or answer == "Y" or answer == "y"
348
349  def DeleteBranch(self, name):
350    for line in self.GitBranch().splitlines():
351      if re.match(r".*\s+%s$" % name, line):
352        msg = "Branch %s exists, do you want to delete it?" % name
353        if self.Confirm(msg):
354          self.GitDeleteBranch(name)
355          print "Branch %s deleted." % name
356        else:
357          msg = "Can't continue. Please delete branch %s and try again." % name
358          self.Die(msg)
359
360  def InitialEnvironmentChecks(self):
361    # Cancel if this is not a git checkout.
362    if not os.path.exists(self._config[DOT_GIT_LOCATION]):  # pragma: no cover
363      self.Die("This is not a git checkout, this script won't work for you.")
364
365    # Cancel if EDITOR is unset or not executable.
366    if (self._options.requires_editor and (not os.environ.get("EDITOR") or
367        Command("which", os.environ["EDITOR"]) is None)):  # pragma: no cover
368      self.Die("Please set your EDITOR environment variable, you'll need it.")
369
370  def CommonPrepare(self):
371    # Check for a clean workdir.
372    if not self.GitIsWorkdirClean():  # pragma: no cover
373      self.Die("Workspace is not clean. Please commit or undo your changes.")
374
375    # Persist current branch.
376    self["current_branch"] = self.GitCurrentBranch()
377
378    # Fetch unfetched revisions.
379    self.GitSVNFetch()
380
381  def PrepareBranch(self):
382    # Delete the branch that will be created later if it exists already.
383    self.DeleteBranch(self._config[BRANCHNAME])
384
385  def CommonCleanup(self):
386    self.GitCheckout(self["current_branch"])
387    if self._config[BRANCHNAME] != self["current_branch"]:
388      self.GitDeleteBranch(self._config[BRANCHNAME])
389
390    # Clean up all temporary files.
391    Command("rm", "-f %s*" % self._config[PERSISTFILE_BASENAME])
392
393  def ReadAndPersistVersion(self, prefix=""):
394    def ReadAndPersist(var_name, def_name):
395      match = re.match(r"^#define %s\s+(\d*)" % def_name, line)
396      if match:
397        value = match.group(1)
398        self["%s%s" % (prefix, var_name)] = value
399    for line in LinesInFile(self._config[VERSION_FILE]):
400      for (var_name, def_name) in [("major", "MAJOR_VERSION"),
401                                   ("minor", "MINOR_VERSION"),
402                                   ("build", "BUILD_NUMBER"),
403                                   ("patch", "PATCH_LEVEL")]:
404        ReadAndPersist(var_name, def_name)
405
406  def WaitForLGTM(self):
407    print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit "
408           "your change. (If you need to iterate on the patch or double check "
409           "that it's sane, do so in another shell, but remember to not "
410           "change the headline of the uploaded CL.")
411    answer = ""
412    while answer != "LGTM":
413      print "> ",
414      answer = self.ReadLine(None if self._options.wait_for_lgtm else "LGTM")
415      if answer != "LGTM":
416        print "That was not 'LGTM'."
417
418  def WaitForResolvingConflicts(self, patch_file):
419    print("Applying the patch \"%s\" failed. Either type \"ABORT<Return>\", "
420          "or resolve the conflicts, stage *all* touched files with "
421          "'git add', and type \"RESOLVED<Return>\"")
422    self.DieNoManualMode()
423    answer = ""
424    while answer != "RESOLVED":
425      if answer == "ABORT":
426        self.Die("Applying the patch failed.")
427      if answer != "":
428        print "That was not 'RESOLVED' or 'ABORT'."
429      print "> ",
430      answer = self.ReadLine()
431
432  # Takes a file containing the patch to apply as first argument.
433  def ApplyPatch(self, patch_file, revert=False):
434    try:
435      self.GitApplyPatch(patch_file, revert)
436    except GitFailedException:
437      self.WaitForResolvingConflicts(patch_file)
438
439  def FindLastTrunkPush(self, parent_hash="", include_patches=False):
440    push_pattern = "^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*"
441    if not include_patches:
442      # Non-patched versions only have three numbers followed by the "(based
443      # on...) comment."
444      push_pattern += " (based"
445    branch = "" if parent_hash else "svn/trunk"
446    return self.GitLog(n=1, format="%H", grep=push_pattern,
447                       parent_hash=parent_hash, branch=branch)
448
449
450class UploadStep(Step):
451  MESSAGE = "Upload for code review."
452
453  def RunStep(self):
454    if self._options.reviewer:
455      print "Using account %s for review." % self._options.reviewer
456      reviewer = self._options.reviewer
457    else:
458      print "Please enter the email address of a V8 reviewer for your patch: ",
459      self.DieNoManualMode("A reviewer must be specified in forced mode.")
460      reviewer = self.ReadLine()
461    self.GitUpload(reviewer, self._options.author, self._options.force_upload)
462
463
464class DetermineV8Sheriff(Step):
465  MESSAGE = "Determine the V8 sheriff for code review."
466
467  def RunStep(self):
468    self["sheriff"] = None
469    if not self._options.sheriff:  # pragma: no cover
470      return
471
472    try:
473      # The googlers mapping maps @google.com accounts to @chromium.org
474      # accounts.
475      googlers = imp.load_source('googlers_mapping',
476                                 self._options.googlers_mapping)
477      googlers = googlers.list_to_dict(googlers.get_list())
478    except:  # pragma: no cover
479      print "Skip determining sheriff without googler mapping."
480      return
481
482    # The sheriff determined by the rotation on the waterfall has a
483    # @google.com account.
484    url = "https://chromium-build.appspot.com/p/chromium/sheriff_v8.js"
485    match = re.match(r"document\.write\('(\w+)'\)", self.ReadURL(url))
486
487    # If "channel is sheriff", we can't match an account.
488    if match:
489      g_name = match.group(1)
490      self["sheriff"] = googlers.get(g_name + "@google.com",
491                                     g_name + "@chromium.org")
492      self._options.reviewer = self["sheriff"]
493      print "Found active sheriff: %s" % self["sheriff"]
494    else:
495      print "No active sheriff found."
496
497
498def MakeStep(step_class=Step, number=0, state=None, config=None,
499             options=None, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
500    # Allow to pass in empty dictionaries.
501    state = state if state is not None else {}
502    config = config if config is not None else {}
503
504    try:
505      message = step_class.MESSAGE
506    except AttributeError:
507      message = step_class.__name__
508    try:
509      requires = step_class.REQUIRES
510    except AttributeError:
511      requires = None
512
513    return step_class(message, requires, number=number, config=config,
514                      state=state, options=options,
515                      handler=side_effect_handler)
516
517
518class ScriptsBase(object):
519  # TODO(machenbach): Move static config here.
520  def __init__(self, config, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER,
521               state=None):
522    self._config = config
523    self._side_effect_handler = side_effect_handler
524    self._state = state if state is not None else {}
525
526  def _Description(self):
527    return None
528
529  def _PrepareOptions(self, parser):
530    pass
531
532  def _ProcessOptions(self, options):
533    return True
534
535  def _Steps(self):  # pragma: no cover
536    raise Exception("Not implemented.")
537
538  def MakeOptions(self, args=None):
539    parser = argparse.ArgumentParser(description=self._Description())
540    parser.add_argument("-a", "--author", default="",
541                        help="The author email used for rietveld.")
542    parser.add_argument("-g", "--googlers-mapping",
543                        help="Path to the script mapping google accounts.")
544    parser.add_argument("-r", "--reviewer", default="",
545                        help="The account name to be used for reviews.")
546    parser.add_argument("--sheriff", default=False, action="store_true",
547                        help=("Determine current sheriff to review CLs. On "
548                              "success, this will overwrite the reviewer "
549                              "option."))
550    parser.add_argument("-s", "--step",
551        help="Specify the step where to start work. Default: 0.",
552        default=0, type=int)
553
554    self._PrepareOptions(parser)
555
556    if args is None:  # pragma: no cover
557      options = parser.parse_args()
558    else:
559      options = parser.parse_args(args)
560
561    # Process common options.
562    if options.step < 0:  # pragma: no cover
563      print "Bad step number %d" % options.step
564      parser.print_help()
565      return None
566    if options.sheriff and not options.googlers_mapping:  # pragma: no cover
567      print "To determine the current sheriff, requires the googler mapping"
568      parser.print_help()
569      return None
570
571    # Defaults for options, common to all scripts.
572    options.manual = getattr(options, "manual", True)
573    options.force = getattr(options, "force", False)
574
575    # Derived options.
576    options.requires_editor = not options.force
577    options.wait_for_lgtm = not options.force
578    options.force_readline_defaults = not options.manual
579    options.force_upload = not options.manual
580
581    # Process script specific options.
582    if not self._ProcessOptions(options):
583      parser.print_help()
584      return None
585    return options
586
587  def RunSteps(self, step_classes, args=None):
588    options = self.MakeOptions(args)
589    if not options:
590      return 1
591
592    state_file = "%s-state.json" % self._config[PERSISTFILE_BASENAME]
593    if options.step == 0 and os.path.exists(state_file):
594      os.remove(state_file)
595
596    steps = []
597    for (number, step_class) in enumerate(step_classes):
598      steps.append(MakeStep(step_class, number, self._state, self._config,
599                            options, self._side_effect_handler))
600    for step in steps[options.step:]:
601      if step.Run():
602        return 1
603    return 0
604
605  def Run(self, args=None):
606    return self.RunSteps(self._Steps(), args)
607