• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- python -*-
2# ex: set syntax=python:
3
4c = BuildmasterConfig = {}
5
6from buildbot.buildslave import BuildSlave
7from buildbot.changes.pb import PBChangeSource
8from buildbot.scheduler import AnyBranchScheduler, Triggerable
9from buildbot.status import html
10from buildbot.process import buildstep, factory, properties
11from buildbot.steps import master, shell, source, transfer, trigger
12from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
13
14from twisted.internet import defer
15
16import re
17import simplejson
18
19WithProperties = properties.WithProperties
20
21class ConfigureBuild(buildstep.BuildStep):
22    name = "configure build"
23    description = ["configuring build"]
24    descriptionDone = ["configured build"]
25    def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs):
26        buildstep.BuildStep.__init__(self, *args, **kwargs)
27        self.platform = platform.split('-', 1)[0]
28        self.fullPlatform = platform
29        self.configuration = configuration
30        self.architecture = architecture
31        self.buildOnly = buildOnly
32        self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly)
33
34    def start(self):
35        self.setProperty("platform", self.platform)
36        self.setProperty("fullPlatform", self.fullPlatform)
37        self.setProperty("configuration", self.configuration)
38        self.setProperty("architecture", self.architecture)
39        self.setProperty("buildOnly", self.buildOnly)
40        self.finished(SUCCESS)
41        return defer.succeed(None)
42
43
44class CheckOutSource(source.SVN):
45    baseURL = "http://svn.webkit.org/repository/webkit/"
46    mode = "update"
47    def __init__(self, *args, **kwargs):
48        source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
49
50
51class InstallWin32Dependencies(shell.Compile):
52    description = ["installing dependencies"]
53    descriptionDone = ["installed dependencies"]
54    command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"]
55
56
57class InstallChromiumDependencies(shell.ShellCommand):
58    name = "gclient"
59    description = ["updating chromium dependencies"]
60    descriptionDone = ["updated chromium dependencies"]
61    command = ["perl", "./WebKitTools/Scripts/update-webkit-chromium"]
62    haltOnFailure = True
63
64
65def appendCustomBuildFlags(step, platform):
66    if platform in ('gtk', 'wx', 'qt', 'chromium'):
67        step.setCommand(step.command + ['--' + platform])
68
69
70class CompileWebKit(shell.Compile):
71    command = ["perl", "./WebKitTools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
72    env = {'MFLAGS':''}
73    name = "compile-webkit"
74    description = ["compiling"]
75    descriptionDone = ["compiled"]
76    warningPattern = ".*arning: .*"
77
78    def start(self):
79        platform = self.getProperty('platform')
80        buildOnly = self.getProperty('buildOnly')
81        if platform == 'mac' and buildOnly:
82            self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
83
84        appendCustomBuildFlags(self, platform)
85        return shell.Compile.start(self)
86
87
88class ArchiveBuiltProduct(shell.ShellCommand):
89    command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive",
90               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
91    name = "archive-built-product"
92    description = ["archiving built product"]
93    descriptionDone = ["archived built product"]
94    haltOnFailure = True
95
96
97class ExtractBuiltProduct(shell.ShellCommand):
98    command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive",
99               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]
100    name = "extract-built-product"
101    description = ["extracting built product"]
102    descriptionDone = ["extracted built product"]
103    haltOnFailure = True
104
105
106class UploadBuiltProduct(transfer.FileUpload):
107    slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
108    masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
109    haltOnFailure = True
110
111    def __init__(self):
112        transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest)
113
114
115class DownloadBuiltProduct(transfer.FileDownload):
116    slavedest = WithProperties("WebKitBuild/%(configuration)s.zip")
117    mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
118    haltOnFailure = True
119
120    def __init__(self):
121        transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest)
122
123
124class RunJavaScriptCoreTests(shell.Test):
125    name = "jscore-test"
126    description = ["jscore-tests running"]
127    descriptionDone = ["jscore-tests"]
128    command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
129    logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'}
130
131    def __init__(self, skipBuild=False, *args, **kwargs):
132        self.skipBuild = skipBuild
133        shell.Test.__init__(self, *args, **kwargs)
134        self.addFactoryArguments(skipBuild=skipBuild)
135
136    def start(self):
137        appendCustomBuildFlags(self, self.getProperty('platform'))
138        if self.skipBuild:
139            self.setCommand(self.command + ['--skip-build'])
140        return shell.Test.start(self)
141
142    def commandComplete(self, cmd):
143        shell.Test.commandComplete(self, cmd)
144
145        logText = cmd.logs['stdio'].getText()
146        statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
147        if statusLines and statusLines[0].split()[0] != '0':
148            self.regressionLine = statusLines[0]
149        else:
150            self.regressionLine = None
151
152    def evaluateCommand(self, cmd):
153        if self.regressionLine:
154            return FAILURE
155
156        if cmd.rc != 0:
157            return FAILURE
158
159        return SUCCESS
160
161    def getText(self, cmd, results):
162        return self.getText2(cmd, results)
163
164    def getText2(self, cmd, results):
165        if results != SUCCESS and self.regressionLine:
166            return [self.name, self.regressionLine]
167
168        return [self.name]
169
170
171class RunWebKitTests(shell.Test):
172    name = "layout-test"
173    description = ["layout-tests running"]
174    descriptionDone = ["layout-tests"]
175    command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results",
176               "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests",
177               WithProperties("--%(configuration)s"), "--exit-after-n-failures", "20"]
178
179    def __init__(self, skipBuild=False, *args, **kwargs):
180        self.skipBuild = skipBuild
181        shell.Test.__init__(self, *args, **kwargs)
182        self.addFactoryArguments(skipBuild=skipBuild)
183
184    def start(self):
185        appendCustomBuildFlags(self, self.getProperty('platform'))
186        if self.skipBuild:
187            self.setCommand(self.command + ['--root=WebKitBuild/bin'])
188        return shell.Test.start(self)
189
190    def commandComplete(self, cmd):
191        shell.Test.commandComplete(self, cmd)
192
193        logText = cmd.logs['stdio'].getText()
194        incorrectLayoutLines = []
195        for line in logText.splitlines():
196            if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
197                incorrectLayoutLines.append(line)
198            elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
199                incorrectLayoutLines.append(line)
200            elif line.startswith("WARNING:") and line.find(' leak') >= 0:
201                incorrectLayoutLines.append(line.replace('WARNING: ', ''))
202            elif line.find('Exiting early') >= 0:
203                incorrectLayoutLines.append(line)
204
205            # FIXME: Detect and summarize leaks of RefCounted objects
206
207        self.incorrectLayoutLines = incorrectLayoutLines
208
209    def evaluateCommand(self, cmd):
210        if self.incorrectLayoutLines:
211            if len(self.incorrectLayoutLines) == 1:
212                line = self.incorrectLayoutLines[0]
213                if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
214                    return WARNINGS
215
216            return FAILURE
217
218        if cmd.rc != 0:
219            return FAILURE
220
221        return SUCCESS
222
223    def getText(self, cmd, results):
224        return self.getText2(cmd, results)
225
226    def getText2(self, cmd, results):
227        if results != SUCCESS and self.incorrectLayoutLines:
228            return self.incorrectLayoutLines
229
230        return [self.name]
231
232
233class RunWebKitLeakTests(RunWebKitTests):
234    def start(self):
235        self.setCommand(self.command + ["--leaks"])
236        return RunWebKitTests.start(self)
237
238
239class ArchiveTestResults(shell.ShellCommand):
240    command = ["python", "./WebKitTools/BuildSlaveSupport/test-result-archive",
241               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
242    name = "archive-test-results"
243    description = ["archiving test results"]
244    descriptionDone = ["archived test results"]
245    haltOnFailure = True
246
247
248class UploadTestResults(transfer.FileUpload):
249    slavesrc = "layout-test-results.zip"
250    masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
251
252    def __init__(self):
253        transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest)
254
255
256class ExtractTestResults(master.MasterShellCommand):
257    zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
258    resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
259
260    def __init__(self):
261        master.MasterShellCommand.__init__(self, "")
262
263    def start(self):
264        self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]
265        return master.MasterShellCommand.start(self)
266
267    def finished(self, result):
268        url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "")
269        self.addURL("view results", url)
270        result = master.MasterShellCommand.finished(self, result)
271        self.step_status.setText(["uploaded results"])
272        return result
273
274
275class Factory(factory.BuildFactory):
276    def __init__(self, platform, configuration, architectures, buildOnly):
277        factory.BuildFactory.__init__(self)
278        self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly)
279        self.addStep(CheckOutSource)
280        if platform == "win":
281            self.addStep(InstallWin32Dependencies)
282        if platform == "chromium":
283            self.addStep(InstallChromiumDependencies)
284
285class BuildFactory(Factory):
286    def __init__(self, platform, configuration, architectures, triggers=None):
287        Factory.__init__(self, platform, configuration, architectures, True)
288        self.addStep(CompileWebKit)
289        if triggers:
290            self.addStep(ArchiveBuiltProduct)
291            self.addStep(UploadBuiltProduct)
292            self.addStep(trigger.Trigger, schedulerNames=triggers)
293
294class TestFactory(Factory):
295    def __init__(self, platform, configuration, architectures):
296        Factory.__init__(self, platform, configuration, architectures, False)
297        self.addStep(DownloadBuiltProduct)
298        self.addStep(ExtractBuiltProduct)
299        self.addStep(RunJavaScriptCoreTests, skipBuild=True)
300        self.addStep(RunWebKitTests, skipBuild=(platform == 'win'))
301        self.addStep(ArchiveTestResults)
302        self.addStep(UploadTestResults)
303        self.addStep(ExtractTestResults)
304
305class BuildAndTestFactory(Factory):
306    TestClass = RunWebKitTests
307    def __init__(self, platform, configuration, architectures):
308        Factory.__init__(self, platform, configuration, architectures, False)
309        self.addStep(CompileWebKit)
310        self.addStep(RunJavaScriptCoreTests)
311        self.addStep(self.TestClass)
312        self.addStep(ArchiveTestResults)
313        self.addStep(UploadTestResults)
314        self.addStep(ExtractTestResults)
315
316class BuildAndTestLeaksFactory(BuildAndTestFactory):
317    TestClass = RunWebKitLeakTests
318
319
320def loadBuilderConfig(c):
321    passwords = simplejson.load(open('passwords.json'))
322
323    config = simplejson.load(open('config.json'))
324
325    c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
326
327    c['schedulers'] = []
328    for scheduler in config['schedulers']:
329        kls = globals()[scheduler.pop('type')]
330        c['schedulers'].append(kls(**scheduler))
331
332    c['builders'] = []
333    for builder in config['builders']:
334        for slaveName in builder['slavenames']:
335            for slave in config['slaves']:
336                if slave['name'] != slaveName or slave['platform'] == '*':
337                    continue
338
339                if slave['platform'] != builder['platform']:
340                    raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
341
342                break
343
344        factory = globals()["%sFactory" % builder.pop('type')]
345        factoryArgs = []
346        for key in "platform", "configuration", "architectures", "triggers":
347            value = builder.pop(key, None)
348            if value:
349                factoryArgs.append(value)
350
351        builder["factory"] = factory(*factoryArgs)
352
353        c['builders'].append(builder)
354
355loadBuilderConfig(c)
356
357c['change_source'] = PBChangeSource()
358
359c['status'] = []
360c['status'].append(html.WebStatus(http_port=8710, allowForce=True))
361
362c['slavePortnum'] = 17000
363c['projectName'] = "WebKit"
364c['projectURL'] = "http://webkit.org"
365c['buildbotURL'] = "http://build.webkit.org/"
366