1# Copyright (c) 2011 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import os 30import optparse 31from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand 32from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory 33from webkitpy.layout_tests.models.test_expectations import TestExpectationParser, TestExpectationsModel, TestExpectations 34from webkitpy.layout_tests.port import builders 35from webkitpy.common.net import sheriff_calendar 36 37 38class FlakyTests(AbstractDeclarativeCommand): 39 name = "update-flaky-tests" 40 help_text = "Update FlakyTests file from the flakiness dashboard" 41 show_in_main_help = True 42 43 ALWAYS_CC = [ 44 'ojan@chromium.org', 45 'dpranke@chromium.org', 46 'eseidel@chromium.org', 47 ] 48 49 COMMIT_MESSAGE = ( 50 'Update FlakyTests to match current flakiness dashboard results\n\n' 51 'Automatically generated using:\n' 52 'webkit-patch update-flaky-tests\n\n' 53 'R=%s\n') 54 55 FLAKY_TEST_CONTENTS = ( 56 '# This file is generated by webkit-patch update-flaky-tests from the flakiness dashboard data.\n' 57 '# Manual changes will be overwritten.\n\n' 58 '%s\n') 59 60 def __init__(self): 61 options = [ 62 optparse.make_option('--upload', action='store_true', 63 help='upload the changed FlakyTest file for review'), 64 optparse.make_option('--reviewers', action='store', 65 help='comma-separated list of reviewers, defaults to blink gardeners'), 66 ] 67 AbstractDeclarativeCommand.__init__(self, options=options) 68 # This is sorta silly, but allows for unit testing: 69 self.expectations_factory = BotTestExpectationsFactory 70 71 def _collect_expectation_lines(self, builder_names, factory): 72 model = TestExpectationsModel() 73 for builder_name in builder_names: 74 expectations = factory.expectations_for_builder(builder_name) 75 for line in expectations.expectation_lines(only_ignore_very_flaky=True): 76 model.add_expectation_line(line) 77 # FIXME: We need an official API to get all the test names or all test lines. 78 return model._test_to_expectation_line.values() 79 80 def _commit_and_upload(self, tool, options): 81 files = tool.scm().changed_files() 82 flaky_tests_path = 'LayoutTests/FlakyTests' 83 if flaky_tests_path not in files: 84 print "%s is not changed, not uploading." % flaky_tests_path 85 return 0 86 87 if options.reviewers: 88 # FIXME: Could validate these as emails. sheriff_calendar has some code for that. 89 reviewer_emails = options.reviewers.split(',') 90 else: 91 reviewer_emails = sheriff_calendar.current_gardener_emails() 92 if not reviewer_emails: 93 print "No gardener, and --reviewers not specified, not bothering." 94 return 1 95 96 commit_message = self.COMMIT_MESSAGE % ','.join(reviewer_emails) 97 git_cmd = ['git', 'commit', '-m', commit_message, 98 tool.filesystem.join(tool.scm().checkout_root, flaky_tests_path)] 99 tool.executive.run_and_throw_if_fail(git_cmd) 100 101 git_cmd = ['git', 'cl', 'upload', '--send-mail', '-f', 102 '--cc', ','.join(self.ALWAYS_CC)] 103 tool.executive.run_and_throw_if_fail(git_cmd) 104 105 def execute(self, options, args, tool): 106 factory = self.expectations_factory() 107 lines = self._collect_expectation_lines(builders.all_builder_names(), factory) 108 lines.sort(key=lambda line: line.path) 109 110 port = tool.port_factory.get() 111 # Skip any tests which are mentioned in the dashboard but not in our checkout: 112 fs = tool.filesystem 113 lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), line.path)), lines) 114 115 # Note: This includes all flaky tests from the dashboard, even ones mentioned 116 # in existing TestExpectations. We could certainly load existing TestExpecations 117 # and filter accordingly, or update existing TestExpectations instead of FlakyTests. 118 flaky_tests_path = fs.join(port.layout_tests_dir(), 'FlakyTests') 119 flaky_tests_contents = self.FLAKY_TEST_CONTENTS % TestExpectations.list_to_string(lines) 120 fs.write_text_file(flaky_tests_path, flaky_tests_contents) 121 print "Updated %s" % flaky_tests_path 122 123 if options.upload: 124 return self._commit_and_upload(tool, options) 125 126