Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 c = BuildmasterConfig = {}
   5
   6 from buildbot.buildslave import BuildSlave
   7 from buildbot.changes.pb import PBChangeSource
   8 from buildbot.scheduler import AnyBranchScheduler, Triggerable
   9 from buildbot.schedulers.filter import ChangeFilter
  10 from buildbot.status import html
  11 from buildbot.status.web.authz import Authz
  12 from buildbot.process import buildstep, factory, properties
  13 from buildbot.steps import master, shell, source, transfer, trigger
  14 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
  15
  16 from twisted.internet import defer
  17
  18 import os
  19 import re
  20 import simplejson
  21 import urllib
  22
  23 from webkitpy.common.config import build as wkbuild
  24 from webkitpy.common.net.buildbot import BuildBot as wkbuildbot
  25
  26 WithProperties = properties.WithProperties
  27
  28 class ConfigureBuild(buildstep.BuildStep):
  29     name = "configure build"
  30     description = ["configuring build"]
  31     descriptionDone = ["configured build"]
  32     def __init__(self, platform, configuration, architecture, buildOnly, features, *args, **kwargs):
  33         buildstep.BuildStep.__init__(self, *args, **kwargs)
  34         self.platform = platform.split('-', 1)[0]
  35         self.fullPlatform = platform
  36         self.configuration = configuration
  37         self.architecture = architecture
  38         self.buildOnly = buildOnly
  39         self.features = features;
  40         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, features=features)
  41
  42     def start(self):
  43         self.setProperty("platform", self.platform)
  44         self.setProperty("fullPlatform", self.fullPlatform)
  45         self.setProperty("configuration", self.configuration)
  46         self.setProperty("architecture", self.architecture)
  47         self.setProperty("buildOnly", self.buildOnly)
  48         self.setProperty("features", self.features)
  49         self.finished(SUCCESS)
  50         return defer.succeed(None)
  51
  52
  53 class CheckOutSource(source.SVN):
  54     baseURL = "http://svn.webkit.org/repository/webkit/"
  55     mode = "update"
  56     def __init__(self, *args, **kwargs):
  57         source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
  58
  59
  60 class InstallWin32Dependencies(shell.Compile):
  61     description = ["installing dependencies"]
  62     descriptionDone = ["installed dependencies"]
  63     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
  64
  65 class KillOldProcesses(shell.Compile):
  66     name = "kill old processes"
  67     description = ["killing old processes"]
  68     descriptionDone = ["killed old processes"]
  69     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
  70
  71 class InstallGtkDependencies(shell.ShellCommand):
  72     name = "jhbuild"
  73     description = ["updating gtk dependencies"]
  74     descriptionDone = ["updated gtk dependencies"]
  75     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
  76     haltOnFailure = True
  77
  78 class InstallChromiumDependencies(shell.ShellCommand):
  79     name = "gclient"
  80     description = ["updating chromium dependencies"]
  81     descriptionDone = ["updated chromium dependencies"]
  82     command = ["perl", "./Tools/Scripts/update-webkit-chromium", "--force"]
  83     haltOnFailure = True
  84     def start(self):
  85         if self.getProperty('fullPlatform') == "chromium-android":
  86             self.setCommand(self.command + ['--chromium-android'])
  87
  88         return shell.ShellCommand.start(self)
  89
  90 class CleanupChromiumCrashLogs(shell.ShellCommand):
  91     name = "cleanup crash logs"
  92     description = ["removing crash logs"]
  93     descriptionDone = ["removed crash logs"]
  94     command = ["python", "./Tools/BuildSlaveSupport/chromium/remove-crash-logs"]
  95     haltOnFailure = False
  96
  97
  98 def appendCustomBuildFlags(step, platform, fullPlatform="", useFullPlatformForChromiumCGMac=False):
  99     if fullPlatform == "chromium-android":
 100         step.setCommand(step.command + ['--chromium-android'])
 101     elif useFullPlatformForChromiumCGMac and fullPlatform.find('-cg-mac') > 0:
 102         step.setCommand(step.command + ['--platform', fullPlatform])
 103         # Remove this code once Skia transition is done
 104     elif platform in ('chromium', 'efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'):
 105         step.setCommand(step.command + ['--' + platform])
 106
 107
 108 class CompileWebKit(shell.Compile):
 109     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
 110     env = {'MFLAGS':''}
 111     name = "compile-webkit"
 112     description = ["compiling"]
 113     descriptionDone = ["compiled"]
 114     warningPattern = ".*arning: .*"
 115
 116     def start(self):
 117         platform = self.getProperty('platform')
 118         buildOnly = self.getProperty('buildOnly')
 119         if platform == 'mac' and buildOnly:
 120             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
 121
 122         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
 123
 124         features = self.getProperty('features')
 125         for feature in features:
 126             self.setCommand(self.command + ['--' + feature])
 127
 128         return shell.Compile.start(self)
 129
 130
 131 class ArchiveBuiltProduct(shell.ShellCommand):
 132     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
 133                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
 134     name = "archive-built-product"
 135     description = ["archiving built product"]
 136     descriptionDone = ["archived built product"]
 137     haltOnFailure = True
 138
 139
 140 class ExtractBuiltProduct(shell.ShellCommand):
 141     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
 142                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]
 143     name = "extract-built-product"
 144     description = ["extracting built product"]
 145     descriptionDone = ["extracted built product"]
 146     haltOnFailure = True
 147
 148
 149 class UploadBuiltProduct(transfer.FileUpload):
 150     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
 151     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s%(extraFeatures)s/%(got_revision)s.zip",
 152                                 extraFeatures=lambda properties: UploadBuiltProduct.determineExtraFeatures(properties))
 153     haltOnFailure = True
 154
 155     def __init__(self):
 156         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
 157
 158     @staticmethod
 159     def determineExtraFeatures(properties):
 160         if not properties.has_key("features"):
 161             return ''
 162         features = properties.getProperty("features")
 163         if not len(features):
 164             return ''
 165         result = '-with-' + '-and-'.join(features)
 166         # Just to make sure no feature will add invalid chars, we allow only letters here.
 167         result = re.sub('[^a-zA-Z]', '-', result)
 168         return result
 169
 170
 171 class DownloadBuiltProduct(transfer.FileDownload):
 172     slavedest = WithProperties("WebKitBuild/%(configuration)s.zip")
 173     mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
 174     haltOnFailure = True
 175     flunkOnFailure = True
 176
 177     def __init__(self):
 178         transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest)
 179
 180
 181 class RunJavaScriptCoreTests(shell.Test):
 182     name = "jscore-test"
 183     description = ["jscore-tests running"]
 184     descriptionDone = ["jscore-tests"]
 185     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
 186     logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}
 187
 188     def __init__(self, buildJSCTool=True, *args, **kwargs):
 189         self.buildJSCTool = buildJSCTool
 190         shell.Test.__init__(self, *args, **kwargs)
 191         self.addFactoryArguments(buildJSCTool=buildJSCTool)
 192
 193     def start(self):
 194         appendCustomBuildFlags(self, self.getProperty('platform'))
 195         if not self.buildJSCTool:
 196             self.setCommand(self.command + ['--no-build'])
 197         return shell.Test.start(self)
 198
 199     def commandComplete(self, cmd):
 200         shell.Test.commandComplete(self, cmd)
 201
 202         logText = cmd.logs['stdio'].getText()
 203         statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
 204         if statusLines and statusLines[0].split()[0] != '0':
 205             self.regressionLine = statusLines[0]
 206         else:
 207             self.regressionLine = None
 208
 209         if 'actual.html (source)' in cmd.logs:
 210             self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())
 211
 212     def evaluateCommand(self, cmd):
 213         if self.regressionLine:
 214             return FAILURE
 215
 216         if cmd.rc != 0:
 217             return FAILURE
 218
 219         return SUCCESS
 220
 221     def getText(self, cmd, results):
 222         return self.getText2(cmd, results)
 223
 224     def getText2(self, cmd, results):
 225         if results != SUCCESS and self.regressionLine:
 226             return [self.name, self.regressionLine]
 227
 228         return [self.name]
 229
 230
 231 class RunWebKitTests(shell.Test):
 232     name = "layout-test"
 233     description = ["layout-tests running"]
 234     descriptionDone = ["layout-tests"]
 235     command = ["perl", "./Tools/Scripts/run-webkit-tests",
 236                "--no-launch-safari",
 237                "--no-new-test-results",
 238                "--no-sample-on-timeout",
 239                "--results-directory", "layout-test-results",
 240                "--use-remote-links-to-tests",
 241                "--builder-name", WithProperties("%(buildername)s"),
 242                "--build-number", WithProperties("%(buildnumber)s"),
 243                "--master-name", "webkit.org",
 244                "--test-results-server", "test-results.appspot.com",
 245                WithProperties("--%(configuration)s"),
 246                "--exit-after-n-crashes-or-timeouts", "20",
 247                "--exit-after-n-failures", "500"]
 248
 249     def __init__(self, buildJSCTool=True, *args, **kwargs):
 250         self.buildJSCTool = buildJSCTool
 251         shell.Test.__init__(self, *args, **kwargs)
 252         self.addFactoryArguments(buildJSCTool=buildJSCTool)
 253
 254     def start(self):
 255         platform = self.getProperty('platform')
 256         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'), True)
 257         if platform == "win":
 258             rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin")]
 259         else:
 260             rootArgument = ['--root=WebKitBuild/bin']
 261         if not self.buildJSCTool:
 262             self.setCommand(self.command + rootArgument)
 263         return shell.Test.start(self)
 264
 265     def _parseOldRunWebKitTestsOutput(self, logText):
 266         incorrectLayoutLines = []
 267         for line in logText.splitlines():
 268             if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
 269                 incorrectLayoutLines.append(line)
 270             elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
 271                 incorrectLayoutLines.append(line)
 272             elif line.startswith("WARNING:") and line.find(' leak') >= 0:
 273                 incorrectLayoutLines.append(line.replace('WARNING: ', ''))
 274             elif line.find('Exiting early') >= 0:
 275                 incorrectLayoutLines.append(line)
 276
 277             # FIXME: Detect and summarize leaks of RefCounted objects
 278
 279         self.incorrectLayoutLines = incorrectLayoutLines
 280
 281     # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
 282     nrwt_log_message_regexp = re.compile(r'(?P<log_prefix>.*) (?P<log_level>DEBUG|INFO) (?P<message>.*)')
 283
 284     def _strip_python_logging_prefix(self, line):
 285         match_object = self.nrwt_log_message_regexp.match(line)
 286         if match_object:
 287             return match_object.group('message')
 288         return line
 289
 290     def _parseNewRunWebKitTestsOutput(self, logText):
 291         incorrectLayoutLines = []
 292         expressions = [
 293             ('flakes', re.compile(r'Unexpected flakiness.+:?\s*\((\d+)\)')),
 294             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
 295             ('missing results', re.compile(r'no expected results found\s*:\s+\((\d+)\)')),
 296             ('failures', re.compile(r'Regressions: Unexpected.+:?\s*\((\d+)\)')),
 297         ]
 298         testFailures = {}
 299
 300         for line in logText.splitlines():
 301             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
 302                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
 303                 continue
 304             for name, expression in expressions:
 305                 match = expression.search(line)
 306
 307                 if match:
 308                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
 309                     break
 310
 311                 # FIXME: Parse file names and put them in results
 312
 313         for name in testFailures:
 314             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
 315
 316         self.incorrectLayoutLines = incorrectLayoutLines
 317
 318     def commandComplete(self, cmd):
 319         shell.Test.commandComplete(self, cmd)
 320
 321         logText = cmd.logs['stdio'].getText()
 322         if logText.find("Collecting tests ...") >= 0:
 323             self._parseNewRunWebKitTestsOutput(logText)
 324         else:
 325             self._parseOldRunWebKitTestsOutput(logText)
 326
 327     def evaluateCommand(self, cmd):
 328         result = SUCCESS
 329
 330         if self.incorrectLayoutLines:
 331             if len(self.incorrectLayoutLines) == 1:
 332                 line = self.incorrectLayoutLines[0]
 333                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
 334                     return WARNINGS
 335
 336             for line in self.incorrectLayoutLines:
 337                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
 338                     result = WARNINGS
 339                 else:
 340                     return FAILURE
 341
 342         if cmd.rc != 0:
 343             return FAILURE
 344
 345         return result
 346
 347     def getText(self, cmd, results):
 348         return self.getText2(cmd, results)
 349
 350     def getText2(self, cmd, results):
 351         if results != SUCCESS and self.incorrectLayoutLines:
 352             return self.incorrectLayoutLines
 353
 354         return [self.name]
 355
 356
 357 class NewRunWebKitTests(RunWebKitTests):
 358     command = ["python", "./Tools/Scripts/new-run-webkit-tests",
 359                "--no-show-results", "--no-new-test-results",
 360                "--verbose", "--results-directory", "layout-test-results",
 361                "--builder-name", WithProperties("%(buildername)s"),
 362                "--build-number", WithProperties("%(buildnumber)s"),
 363                "--master-name", "webkit.org",
 364                "--test-results-server", "test-results.appspot.com",
 365                WithProperties("--%(configuration)s")]
 366
 367     def commandComplete(self, cmd):
 368         shell.Test.commandComplete(self, cmd)
 369
 370         logText = cmd.logs['stdio'].getText()
 371         self._parseNewRunWebKitTestsOutput(logText)
 372
 373 class RunUnitTests(shell.Test):
 374     name = "run-api-tests"
 375     description = ["unit tests running"]
 376     descriptionDone = ["unit-tests"]
 377     command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
 378
 379     def start(self):
 380         platform = self.getProperty('platform')
 381         if platform == 'win':
 382             self.setCommand(self.command + ['--no-build'])
 383         if platform.startswith('chromium'):
 384             self.setCommand(self.command + ['--chromium'])
 385         return shell.Test.start(self)
 386
 387
 388 class TestWithFailureCount(shell.Test):
 389     failedTestsFormatString = "%d tests failed"
 390
 391     def countFailures(self, cmd):
 392         return 0
 393
 394     def commandComplete(self, cmd):
 395         shell.Test.commandComplete(self, cmd)
 396         self.failedTestCount = self.countFailures(cmd)
 397
 398     def evaluateCommand(self, cmd):
 399         if self.failedTestCount:
 400             return FAILURE
 401
 402         if cmd.rc != 0:
 403             return FAILURE
 404
 405         return SUCCESS
 406
 407     def getText(self, cmd, results):
 408         return self.getText2(cmd, results)
 409
 410     def getText2(self, cmd, results):
 411         if results != SUCCESS and self.failedTestCount:
 412             return [self.failedTestsFormatString % self.failedTestCount]
 413
 414         return [self.name]
 415
 416
 417 class RunPythonTests(TestWithFailureCount):
 418     name = "webkitpy-test"
 419     description = ["python-tests running"]
 420     descriptionDone = ["python-tests"]
 421     command = ["python", "./Tools/Scripts/test-webkitpy"]
 422     failedTestsFormatString = "%d python tests failed"
 423
 424     def countFailures(self, cmd):
 425         logText = cmd.logs['stdio'].getText()
 426         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
 427         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
 428         for line in logText.splitlines():
 429             match = regex.match(line)
 430             if not match:
 431                 continue
 432             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
 433         return 0
 434
 435
 436 class RunPerlTests(TestWithFailureCount):
 437     name = "webkitperl-test"
 438     description = ["perl-tests running"]
 439     descriptionDone = ["perl-tests"]
 440     command = ["perl", "./Tools/Scripts/test-webkitperl"]
 441     failedTestsFormatString = "%d perl tests failed"
 442
 443     def countFailures(self, cmd):
 444         logText = cmd.logs['stdio'].getText()
 445         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
 446         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
 447         for line in logText.splitlines():
 448             match = regex.match(line)
 449             if not match:
 450                 continue
 451             return int(match.group('count'))
 452         return 0
 453
 454
 455 class RunBindingsTests(shell.Test):
 456     name = "bindings-generation-tests"
 457     description = ["bindings-tests running"]
 458     descriptionDone = ["bindings-tests"]
 459     command = ["python", "./Tools/Scripts/run-bindings-tests"]
 460
 461
 462 class RunGtkAPITests(shell.Test):
 463     name = "API tests"
 464     description = ["API tests running"]
 465     descriptionDone = ["API tests"]
 466     command = ["perl", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]
 467
 468     def commandComplete(self, cmd):
 469         shell.Test.commandComplete(self, cmd)
 470
 471         logText = cmd.logs['stdio'].getText()
 472         incorrectLines = []
 473         for line in logText.splitlines():
 474             if line.startswith('ERROR'):
 475                 incorrectLines.append(line)
 476
 477         self.incorrectLines = incorrectLines
 478
 479     def evaluateCommand(self, cmd):
 480         if self.incorrectLines:
 481             return FAILURE
 482
 483         if cmd.rc != 0:
 484             return FAILURE
 485
 486         return SUCCESS
 487
 488     def getText(self, cmd, results):
 489         return self.getText2(cmd, results)
 490
 491     def getText2(self, cmd, results):
 492         if results != SUCCESS and self.incorrectLines:
 493             return ["%d API tests failed" % len(self.incorrectLines)]
 494
 495         return [self.name]
 496
 497 class RunQtAPITests(shell.Test):
 498     name = "API tests"
 499     description = ["API tests running"]
 500     descriptionDone = ["API tests"]
 501     command = ["python", "./Tools/Scripts/run-qtwebkit-tests",
 502                "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120",
 503                WithProperties("WebKitBuild/%(configuration_pretty)s/Source/WebKit/qt/tests/")]
 504
 505     def start(self):
 506         self.setProperty("configuration_pretty", self.getProperty("configuration").title())
 507         return shell.Test.start(self)
 508
 509     def commandComplete(self, cmd):
 510         shell.Test.commandComplete(self, cmd)
 511
 512         logText = cmd.logs['stdio'].getText()
 513         foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped", logText)
 514
 515         self.incorrectTests = 0
 516         self.statusLine = []
 517
 518         if foundItems:
 519             self.incorrectTests = int(foundItems[0][1])
 520             if self.incorrectTests > 0:
 521                 self.statusLine = [
 522                     "%s passed, %s failed, %s skipped" % (foundItems[0][0], foundItems[0][1], foundItems[0][2])
 523                 ]
 524
 525     def evaluateCommand(self, cmd):
 526         if self.incorrectTests:
 527             return WARNINGS
 528
 529         if cmd.rc != 0:
 530             return FAILURE
 531
 532         return SUCCESS
 533
 534     def getText(self, cmd, results):
 535         return self.getText2(cmd, results)
 536
 537     def getText2(self, cmd, results):
 538         if results != SUCCESS and self.incorrectTests:
 539             return self.statusLine
 540
 541         return [self.name]
 542
 543 class RunWebKitLeakTests(RunWebKitTests):
 544     warnOnWarnings = True
 545     def start(self):
 546         self.setCommand(self.command + ["--leaks"])
 547         return RunWebKitTests.start(self)
 548
 549
 550 class RunWebKit2Tests(RunWebKitTests):
 551     def start(self):
 552         self.setCommand(self.command + ["--webkit-test-runner"])
 553         return RunWebKitTests.start(self)
 554
 555
 556 class RunChromiumWebKitUnitTests(shell.Test):
 557     name = "webkit-unit-tests"
 558     description = ["webkit-unit-tests running"]
 559     descriptionDone = ["webkit-unit-tests"]
 560     command = ["perl", "./Tools/Scripts/run-chromium-webkit-unit-tests",
 561                WithProperties("--%(configuration)s")]
 562
 563
 564 class ArchiveTestResults(shell.ShellCommand):
 565     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
 566                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
 567     name = "archive-test-results"
 568     description = ["archiving test results"]
 569     descriptionDone = ["archived test results"]
 570     haltOnFailure = True
 571
 572
 573 class UploadTestResults(transfer.FileUpload):
 574     slavesrc = "layout-test-results.zip"
 575     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
 576
 577     def __init__(self):
 578         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
 579
 580
 581 class ExtractTestResults(master.MasterShellCommand):
 582     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
 583     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
 584     descriptionDone = ["uploaded results"]
 585
 586     def __init__(self):
 587         master.MasterShellCommand.__init__(self, "")
 588
 589     def resultDirectoryURL(self):
 590         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
 591
 592     def start(self):
 593         self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]
 594         return master.MasterShellCommand.start(self)
 595
 596     def addCustomURLs(self):
 597         url = self.resultDirectoryURL() + "results.html"
 598         self.addURL("view results", url)
 599
 600     def finished(self, result):
 601         self.addCustomURLs()
 602         return master.MasterShellCommand.finished(self, result)
 603
 604
 605 class ExtractTestResultsAndLeaks(ExtractTestResults):
 606     def addCustomURLs(self):
 607         ExtractTestResults.addCustomURLs(self)
 608         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
 609         self.addURL("view leaks", url)
 610
 611
 612 class Factory(factory.BuildFactory):
 613     def __init__(self, platform, configuration, architectures, buildOnly, features=None, **kwargs):
 614         factory.BuildFactory.__init__(self)
 615         self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, features=features)
 616         self.addStep(CheckOutSource)
 617         self.addStep(KillOldProcesses)
 618         if platform == "win":
 619             self.addStep(InstallWin32Dependencies)
 620         if platform.startswith("chromium"):
 621             self.addStep(InstallChromiumDependencies)
 622         if platform == "gtk":
 623             self.addStep(InstallGtkDependencies)
 624
 625
 626 class BuildFactory(Factory):
 627     def __init__(self, platform, configuration, architectures, triggers=None, upload=False, **kwargs):
 628         Factory.__init__(self, platform, configuration, architectures, True, **kwargs)
 629         self.addStep(CompileWebKit)
 630         if triggers or upload:
 631             self.addStep(ArchiveBuiltProduct)
 632             self.addStep(UploadBuiltProduct)
 633         if triggers:
 634             self.addStep(trigger.Trigger, schedulerNames=triggers)
 635
 636 def unitTestsSupported(configuration, platform):
 637     return (platform == 'win' or platform.startswith('mac')
 638             or (platform.startswith('chromium') and platform != 'chromium-android'))
 639
 640 class TestFactory(Factory):
 641     TestClass = RunWebKitTests
 642     ExtractTestResultsClass = ExtractTestResults
 643     def __init__(self, platform, configuration, architectures, **kwargs):
 644         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
 645         self.addStep(DownloadBuiltProduct)
 646         self.addStep(ExtractBuiltProduct)
 647         self.addStep(RunJavaScriptCoreTests, buildJSCTool=False)
 648         self.addStep(self.TestClass, buildJSCTool=(platform != 'win'))
 649
 650         if unitTestsSupported(configuration, platform):
 651             self.addStep(RunUnitTests)
 652         self.addStep(RunPythonTests)
 653         self.addStep(RunPerlTests)
 654         self.addStep(RunBindingsTests)
 655         self.addStep(ArchiveTestResults)
 656         self.addStep(UploadTestResults)
 657         self.addStep(self.ExtractTestResultsClass)
 658
 659 class BuildAndTestFactory(Factory):
 660     TestClass = RunWebKitTests
 661     ExtractTestResultsClass = ExtractTestResults
 662     def __init__(self, platform, configuration, architectures, **kwargs):
 663         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
 664         if platform.startswith("chromium"):
 665             self.addStep(CleanupChromiumCrashLogs)
 666         self.addStep(CompileWebKit)
 667         if not platform.startswith("chromium"):
 668             self.addStep(RunJavaScriptCoreTests)
 669         if platform.startswith("chromium"):
 670             self.addStep(RunChromiumWebKitUnitTests)
 671         self.addStep(self.TestClass)
 672         if unitTestsSupported(configuration, platform):
 673             self.addStep(RunUnitTests)
 674         self.addStep(RunPythonTests)
 675         # Chromium Win runs in non-Cygwin environment, which is not yet fit
 676         # for running tests. This can be removed once bug 48166 is fixed.
 677         if platform != "chromium-win":
 678             self.addStep(RunPerlTests)
 679             self.addStep(RunBindingsTests)
 680         self.addStep(ArchiveTestResults)
 681         self.addStep(UploadTestResults)
 682         self.addStep(self.ExtractTestResultsClass)
 683         if platform == "gtk":
 684             self.addStep(RunGtkAPITests)
 685         if platform == "qt":
 686             self.addStep(RunQtAPITests)
 687
 688 class BuildAndTestLeaksFactory(BuildAndTestFactory):
 689     TestClass = RunWebKitLeakTests
 690     ExtractTestResultsClass = ExtractTestResultsAndLeaks
 691
 692 class NewBuildAndTestFactory(BuildAndTestFactory):
 693     TestClass = NewRunWebKitTests
 694
 695 class TestWebKit2Factory(TestFactory):
 696     TestClass = RunWebKit2Tests
 697
 698 class PlatformSpecificScheduler(AnyBranchScheduler):
 699     def __init__(self, platform, branch, **kwargs):
 700         self.platform = platform
 701         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
 702         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
 703
 704     def filter(self, change):
 705         return wkbuild.should_build(self.platform, change.files)
 706
 707 trunk_filter = ChangeFilter(branch=["trunk", None])
 708
 709 def loadBuilderConfig(c):
 710     # FIXME: These file handles are leaked.
 711     passwords = simplejson.load(open('passwords.json'))
 712     config = simplejson.load(open('config.json'))
 713
 714     # use webkitpy's buildbot module to test for core builders
 715     wkbb = wkbuildbot()
 716
 717     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
 718
 719     c['schedulers'] = []
 720     for scheduler in config['schedulers']:
 721         if "change_filter" in scheduler:
 722             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
 723         kls = globals()[scheduler.pop('type')]
 724         # Python 2.6 can't handle unicode keys as keyword arguments:
 725         # http://bugs.python.org/issue2646.  Modern versions of simplejson return
 726         # unicode strings from simplejson.load, so we map all keys to str objects.
 727         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
 728
 729         # BaseScheduler asserts if given unicode objects instead of strs.
 730         # http://trac.buildbot.net/ticket/2075
 731         scheduler['builderNames'] = map(str, scheduler['builderNames'])
 732         c['schedulers'].append(kls(**scheduler))
 733
 734     c['builders'] = []
 735     for builder in config['builders']:
 736         for slaveName in builder['slavenames']:
 737             for slave in config['slaves']:
 738                 if slave['name'] != slaveName or slave['platform'] == '*':
 739                     continue
 740
 741                 if slave['platform'] != builder['platform']:
 742                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
 743
 744                 break
 745
 746         factory = globals()["%sFactory" % builder.pop('type')]
 747         factoryArgs = []
 748         for key in "platform", "configuration", "architectures", "triggers":
 749             value = builder.pop(key, None)
 750             if value:
 751                 factoryArgs.append(value)
 752
 753         factoryKwArgs = {
 754             "features": builder.pop("features", []),
 755             "upload": builder.pop("upload", False)
 756         }
 757
 758         builder["factory"] = factory(*factoryArgs, **factoryKwArgs)
 759
 760         builder["category"] = "noncore"
 761         if wkbb._is_core_builder(builder['name']):
 762             builder["category"] = "core"
 763
 764         c['builders'].append(builder)
 765
 766 loadBuilderConfig(c)
 767
 768 c['change_source'] = PBChangeSource()
 769
 770 # permissions for WebStatus
 771 authz = Authz(
 772     forceBuild=False,
 773     forceAllBuilds=False,
 774     pingBuilder=True,
 775     gracefulShutdown=False,
 776     stopBuild=True,
 777     stopAllBuilds=True,
 778     cancelPendingBuild=True,
 779     stopChange=True,
 780     cleanShutdown=False)
 781
 782 c['status'] = []
 783 c['status'].append(html.WebStatus(http_port=8710,
 784                                   revlink="http://trac.webkit.org/changeset/%s",
 785                                   authz=authz))
 786
 787 c['slavePortnum'] = 17000
 788 c['projectName'] = "WebKit"
 789 c['projectURL'] = "http://webkit.org"
 790 c['buildbotURL'] = "http://build.webkit.org/"
 791
 792 c['buildHorizon'] = 1000
 793 c['logHorizon'] = 500
 794 c['eventHorizon'] = 200
 795 c['buildCacheSize'] = 60