• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2009, Google Inc. All rights reserved.
2# Copyright (c) 2009 Apple Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29#
30# WebKit's Python module for interacting with Bugzilla
31
32import getpass
33import platform
34import re
35import subprocess
36import urllib2
37
38from datetime import datetime # used in timestamp()
39
40# Import WebKit-specific modules.
41from modules.logging import error, log
42from modules.committers import CommitterList
43
44# WebKit includes a built copy of BeautifulSoup in Scripts/modules
45# so this import should always succeed.
46from .BeautifulSoup import BeautifulSoup
47
48try:
49    from mechanize import Browser
50except ImportError, e:
51    print """
52mechanize is required.
53
54To install:
55sudo easy_install mechanize
56
57Or from the web:
58http://wwwsearch.sourceforge.net/mechanize/
59"""
60    exit(1)
61
62def credentials_from_git():
63    return [read_config("username"), read_config("password")]
64
65def credentials_from_keychain(username=None):
66    if not is_mac_os_x():
67        return [username, None]
68
69    command = "/usr/bin/security %s -g -s %s" % ("find-internet-password", Bugzilla.bug_server_host)
70    if username:
71        command += " -a %s" % username
72
73    log('Reading Keychain for %s account and password.  Click "Allow" to continue...' % Bugzilla.bug_server_host)
74    keychain_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
75    value = keychain_process.communicate()[0]
76    exit_code = keychain_process.wait()
77
78    if exit_code:
79        return [username, None]
80
81    match = re.search('^\s*"acct"<blob>="(?P<username>.+)"', value, re.MULTILINE)
82    if match:
83        username = match.group('username')
84
85    password = None
86    match = re.search('^password: "(?P<password>.+)"', value, re.MULTILINE)
87    if match:
88        password = match.group('password')
89
90    return [username, password]
91
92def is_mac_os_x():
93    return platform.mac_ver()[0]
94
95# FIXME: This should not depend on git for config storage
96def read_config(key):
97    # Need a way to read from svn too
98    config_process = subprocess.Popen("git config --get bugzilla." + key, stdout=subprocess.PIPE, shell=True)
99    value = config_process.communicate()[0]
100    return_code = config_process.wait()
101
102    if return_code:
103        return None
104    return value.rstrip('\n')
105
106def read_credentials():
107    (username, password) = credentials_from_git()
108
109    if not username or not password:
110        (username, password) = credentials_from_keychain(username)
111
112    if not username:
113        username = raw_input("Bugzilla login: ")
114    if not password:
115        password = getpass.getpass("Bugzilla password for %s: " % username)
116
117    return [username, password]
118
119def timestamp():
120    return datetime.now().strftime("%Y%m%d%H%M%S")
121
122class Bugzilla:
123    def __init__(self, dryrun=False, committers=CommitterList()):
124        self.dryrun = dryrun
125        self.authenticated = False
126
127        self.browser = Browser()
128        # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
129        self.browser.set_handle_robots(False)
130        self.committers = committers
131
132    # Defaults (until we support better option parsing):
133    bug_server_host = "bugs.webkit.org"
134    bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
135    bug_server_url = "https://%s/" % bug_server_host
136
137    def bug_url_for_bug_id(self, bug_id, xml=False):
138        content_type = "&ctype=xml" if xml else ""
139        return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
140
141    def attachment_url_for_id(self, attachment_id, action="view"):
142        action_param = ""
143        if action and action != "view":
144            action_param = "&action=" + action
145        return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
146
147    def _parse_attachment_element(self, element, bug_id):
148        attachment = {}
149        attachment['bug_id'] = bug_id
150        attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
151        attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
152        attachment['id'] = str(element.find('attachid').string)
153        attachment['url'] = self.attachment_url_for_id(attachment['id'])
154        attachment['name'] = unicode(element.find('desc').string)
155        attachment['type'] = str(element.find('type').string)
156
157        review_flag = element.find('flag', attrs={"name" : "review"})
158        if review_flag and review_flag['status'] == '+':
159            reviewer_email = review_flag['setter']
160            reviewer = self.committers.reviewer_by_bugzilla_email(reviewer_email)
161            attachment['reviewer'] = reviewer.full_name
162
163        commit_queue_flag = element.find('flag', attrs={"name" : "commit-queue"})
164        if commit_queue_flag and commit_queue_flag['status'] == '+':
165            committer_email = commit_queue_flag['setter']
166            committer = self.committers.committer_by_bugzilla_email(committer_email)
167            attachment['commit-queue'] = committer.full_name
168
169        return attachment
170
171    def fetch_attachments_from_bug(self, bug_id):
172        bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
173        log("Fetching: " + bug_url)
174
175        page = urllib2.urlopen(bug_url)
176        soup = BeautifulSoup(page)
177
178        attachments = []
179        for element in soup.findAll('attachment'):
180            attachment = self._parse_attachment_element(element, bug_id)
181            attachments.append(attachment)
182        return attachments
183
184    def fetch_patches_from_bug(self, bug_id):
185        patches = []
186        for attachment in self.fetch_attachments_from_bug(bug_id):
187            if attachment['is_patch'] and not attachment['is_obsolete']:
188                patches.append(attachment)
189        return patches
190
191    def fetch_reviewed_patches_from_bug(self, bug_id):
192        reviewed_patches = []
193        for attachment in self.fetch_attachments_from_bug(bug_id):
194            if 'reviewer' in attachment and not attachment['is_obsolete']:
195                reviewed_patches.append(attachment)
196        return reviewed_patches
197
198    def fetch_commit_queue_patches_from_bug(self, bug_id):
199        commit_queue_patches = []
200        for attachment in self.fetch_reviewed_patches_from_bug(bug_id):
201            if 'commit-queue' in attachment and not attachment['is_obsolete']:
202                commit_queue_patches.append(attachment)
203        return commit_queue_patches
204
205    def fetch_bug_ids_from_commit_queue(self):
206        commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B"
207
208        page = urllib2.urlopen(commit_queue_url)
209        soup = BeautifulSoup(page)
210
211        bug_ids = []
212        # Grab the cells in the first column (which happens to be the bug ids)
213        for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child"
214            bug_link = bug_link_cell.find("a")
215            bug_ids.append(bug_link.string) # the contents happen to be the bug id
216
217        return bug_ids
218
219    def fetch_patches_from_commit_queue(self):
220        patches_to_land = []
221        for bug_id in self.fetch_bug_ids_from_commit_queue():
222            patches = self.fetch_commit_queue_patches_from_bug(bug_id)
223            patches_to_land += patches
224        return patches_to_land
225
226    def authenticate(self):
227        if self.authenticated:
228            return
229
230        if self.dryrun:
231            log("Skipping log in for dry run...")
232            self.authenticated = True
233            return
234
235        (username, password) = read_credentials()
236
237        log("Logging in as %s..." % username)
238        self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
239        self.browser.select_form(name="login")
240        self.browser['Bugzilla_login'] = username
241        self.browser['Bugzilla_password'] = password
242        response = self.browser.submit()
243
244        match = re.search("<title>(.+?)</title>", response.read())
245        # If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
246        if match and re.search("Invalid", match.group(1), re.IGNORECASE):
247            # FIXME: We could add the ability to try again on failure.
248            raise ScriptError("Bugzilla login failed: %s" % match.group(1))
249
250        self.authenticated = True
251
252    def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False):
253        self.authenticate()
254
255        log('Adding patch "%s" to bug %s' % (description, bug_id))
256        if self.dryrun:
257            log(comment_text)
258            return
259
260        self.browser.open(self.bug_server_url + "attachment.cgi?action=enter&bugid=" + bug_id)
261        self.browser.select_form(name="entryform")
262        self.browser['description'] = description
263        self.browser['ispatch'] = ("1",)
264        if comment_text:
265            log(comment_text)
266            self.browser['comment'] = comment_text
267        self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
268        self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp()))
269        self.browser.submit()
270
271    def prompt_for_component(self, components):
272        log("Please pick a component:")
273        i = 0
274        for name in components:
275            i += 1
276            log("%2d. %s" % (i, name))
277        result = int(raw_input("Enter a number: ")) - 1
278        return components[result]
279
280    def _check_create_bug_response(self, response_html):
281        match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html)
282        if match:
283            return match.group('bug_id')
284
285        match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL)
286        error_message = "FAIL"
287        if match:
288            text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True)
289            error_message = "\n" + '\n'.join(["  " + line.strip() for line in text_lines if line.strip()])
290        raise ScriptError("Bug not created: %s" % error_message)
291
292    def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False):
293        self.authenticate()
294
295        log('Creating bug with patch description "%s"' % patch_description)
296        if self.dryrun:
297            log(bug_description)
298            return
299
300        self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
301        self.browser.select_form(name="Create")
302        component_items = self.browser.find_control('component').items
303        component_names = map(lambda item: item.name, component_items)
304        if not component or component not in component_names:
305            component = self.prompt_for_component(component_names)
306        self.browser['component'] = [component]
307        self.browser['cc'] = cc
308        self.browser['short_desc'] = bug_title
309        if bug_description:
310            log(bug_description)
311            self.browser['comment'] = bug_description
312        self.browser['description'] = patch_description
313        self.browser['ispatch'] = ("1",)
314        self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
315        self.browser.add_file(patch_file_object, "text/plain", "%s.patch" % timestamp(), 'data')
316        response = self.browser.submit()
317
318        bug_id = self._check_create_bug_response(response.read())
319        log("Bug %s created." % bug_id)
320        log(self.bug_server_url + "show_bug.cgi?id=" + bug_id)
321        return bug_id
322
323    def clear_attachment_review_flag(self, attachment_id, additional_comment_text=None):
324        self.authenticate()
325
326        comment_text = "Clearing review flag on attachment: %s" % attachment_id
327        if additional_comment_text:
328            comment_text += "\n\n" + additional_comment_text
329        log(comment_text)
330
331        if self.dryrun:
332            return
333
334        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
335        self.browser.select_form(nr=1)
336        self.browser.set_value(comment_text, name='comment', nr=0)
337        self.browser.find_control(type='select', nr=0).value = ("X",)
338        self.browser.submit()
339
340    def obsolete_attachment(self, attachment_id, comment_text = None):
341        self.authenticate()
342
343        log("Obsoleting attachment: %s" % attachment_id)
344        if self.dryrun:
345            log(comment_text)
346            return
347
348        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
349        self.browser.select_form(nr=1)
350        self.browser.find_control('isobsolete').items[0].selected = True
351        # Also clear any review flag (to remove it from review/commit queues)
352        self.browser.find_control(type='select', nr=0).value = ("X",)
353        if comment_text:
354            log(comment_text)
355            # Bugzilla has two textareas named 'comment', one is somehow hidden.  We want the first.
356            self.browser.set_value(comment_text, name='comment', nr=0)
357        self.browser.submit()
358
359    def post_comment_to_bug(self, bug_id, comment_text):
360        self.authenticate()
361
362        log("Adding comment to bug %s" % bug_id)
363        if self.dryrun:
364            log(comment_text)
365            return
366
367        self.browser.open(self.bug_url_for_bug_id(bug_id))
368        self.browser.select_form(name="changeform")
369        self.browser['comment'] = comment_text
370        self.browser.submit()
371
372    def close_bug_as_fixed(self, bug_id, comment_text=None):
373        self.authenticate()
374
375        log("Closing bug %s as fixed" % bug_id)
376        if self.dryrun:
377            log(comment_text)
378            return
379
380        self.browser.open(self.bug_url_for_bug_id(bug_id))
381        self.browser.select_form(name="changeform")
382        if comment_text:
383            log(comment_text)
384            self.browser['comment'] = comment_text
385        self.browser['bug_status'] = ['RESOLVED']
386        self.browser['resolution'] = ['FIXED']
387        self.browser.submit()
388