perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             ['ryujit'].each { jit ->
  33                 if (arch == 'x64' && jit == 'legacy_backend') {
  34                     return
  35                 }
  36
  37                 ['full_opt', 'min_opt'].each { opt_level ->
  38
  39                     def architecture = arch
  40                     def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  41                     def testEnv = ""
  42
  43                     if (jit == 'legacy_backend') {
  44                         testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
  45                     }
  46
  47                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  48                         // Set the label.
  49                         label('windows_server_2016_clr_perf')
  50                         wrappers {
  51                             credentialsBinding {
  52                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  53                             }
  54                         }
  55
  56                         if (isPR) {
  57                             parameters {
  58                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  59                             }
  60                         }
  61
  62                         parameters {
  63                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  64                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  65                         }
  66
  67                         def configuration = 'Release'
  68                         def runType = isPR ? 'private' : 'rolling'
  69                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  70                         def uploadString = '-uploadToBenchview'
  71
  72                         steps {
  73                             // Batch
  74
  75                             batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  76                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  77                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  78                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  79                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  80                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  81                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  82                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  83                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  84                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  85                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  86                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  87
  88                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  89
  90                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
  91
  92                             // Run with just stopwatch: Profile=Off
  93                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  94                             batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\Off\\")
  95
  96                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  97                             batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\")
  98
  99                             // Run with the full set of counters enabled: Profile=On
 100                             if (opt_level != 'min_opt') {
 101                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 102                                 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
 103
 104                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 105                                 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
 106                             }
 107                         }
 108                     }
 109
 110                     def archiveSettings = new ArchivalSettings()
 111                     archiveSettings.addFiles('bin/toArchive/**')
 112                     archiveSettings.addFiles('machinedata.json')
 113
 114                     Utilities.addArchival(newJob, archiveSettings)
 115                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 116
 117                     newJob.with {
 118                         logRotator {
 119                             artifactDaysToKeep(30)
 120                             daysToKeep(30)
 121                             artifactNumToKeep(200)
 122                             numToKeep(200)
 123                         }
 124                         wrappers {
 125                             timeout {
 126                                 absolute(240)
 127                             }
 128                         }
 129                     }
 130
 131                     if (isPR) {
 132                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 133                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 134
 135                         def opts = ""
 136                         if (opt_level == 'min_opt') {
 137                             opts = '\\W+min_opts'
 138                         }
 139                         def jitt = ""
 140                         if (jit != 'ryujit') {
 141                             jitt = "\\W+${jit}"
 142                         }
 143
 144                         builder.triggerOnlyOnComment()
 145                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 146                         builder.triggerForBranch(branch)
 147                         builder.emitTrigger(newJob)
 148                     }
 149                     else {
 150                         // Set a push trigger
 151                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 152                         builder.emitTrigger(newJob)
 153                     }
 154                 }
 155             }
 156         }
 157     }
 158 }
 159
 160 // Setup throughput perflab tests runs
 161 [true, false].each { isPR ->
 162     ['Windows_NT'].each { os ->
 163         ['x64', 'x86'].each { arch ->
 164             ['ryujit', 'legacy_backend'].each { jit ->
 165                 [true, false].each { pgo_optimized ->
 166                     if (arch == 'x64' && jit == 'legacy_backend') {
 167                         return
 168                     }
 169
 170                     // pgo not supported for legacy_backend
 171                     if (pgo_optimized && jit == 'legacy_backend') {
 172                         return
 173                     }
 174
 175                     ['full_opt', 'min_opt'].each { opt_level ->
 176                         def architecture = arch
 177
 178                         pgo_build = ""
 179                         pgo_test = ""
 180                         pgo_string = "pgo"
 181                         if (!pgo_optimized) {
 182                             pgo_build = " -nopgooptimize"
 183                             pgo_test = " -nopgo"
 184                             pgo_string = "nopgo"
 185                         }
 186
 187                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 188                             // Set the label.
 189                             label('windows_server_2016_clr_perf')
 190                             wrappers {
 191                                 credentialsBinding {
 192                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 193                                 }
 194                             }
 195
 196                             if (isPR) {
 197                                 parameters {
 198                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 199                                 }
 200                             }
 201
 202                             def configuration = 'Release'
 203                             def runType = isPR ? 'private' : 'rolling'
 204                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 205
 206                             steps {
 207                                 // Batch
 208                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 209                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 210                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 211                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 212                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 213                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 214                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 215                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 216                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 217                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 218                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 219                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 220                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 221                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 222                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 223                             }
 224                         }
 225
 226                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 227                         def archiveSettings = new ArchivalSettings()
 228                         archiveSettings.addFiles('throughput-*.csv')
 229                         Utilities.addArchival(newJob, archiveSettings)
 230
 231                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 232
 233                         if (isPR) {
 234                             def opts = ""
 235                             if (opt_level == 'min_opt') {
 236                                 opts = '\\W+min_opts'
 237                             }
 238
 239                             def jitt = ""
 240                             if (jit != 'ryujit') {
 241                                 jitt = "\\W+${jit}"
 242                             }
 243
 244                             def pgo_trigger = ""
 245                             if (pgo_optimized) {
 246                                 pgo_trigger = "\\W+nopgo"
 247                             }
 248
 249
 250                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 251                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 252                             builder.triggerOnlyOnComment()
 253                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 254                             builder.triggerForBranch(branch)
 255                             builder.emitTrigger(newJob)
 256                         }
 257                         else {
 258                             // Set a push trigger
 259                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 260                             builder.emitTrigger(newJob)
 261                         }
 262                     }
 263                 }
 264             }
 265         }
 266     }
 267 }
 268
 269 def static getFullPerfJobName(def project, def os, def isPR) {
 270     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 271 }
 272
 273 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 274 [true, false].each { isPR ->
 275     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 276     def architecture = 'x64'
 277     def configuration = 'Release'
 278
 279     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 280     ['RHEL7.2'].each { os ->
 281         def newBuildJob = job(fullBuildJobName) {
 282             steps {
 283                 shell("./build.sh verbose ${architecture} ${configuration}")
 284             }
 285         }
 286         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 287         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 288         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 289     }
 290
 291
 292     // Actual perf testing on the following OSes
 293     def perfOSList = ['Ubuntu14.04']
 294     perfOSList.each { os ->
 295         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 296
 297             label('linux_clr_perf')
 298             wrappers {
 299                 credentialsBinding {
 300                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 301                 }
 302             }
 303
 304             if (isPR) {
 305                 parameters {
 306                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 307                 }
 308             }
 309
 310             parameters {
 311                 // Cap the maximum number of iterations to 21.
 312                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 313                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 314                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 315             }
 316
 317             def osGroup = getOSGroup(os)
 318             def runType = isPR ? 'private' : 'rolling'
 319             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 320
 321             steps {
 322                 shell("./tests/scripts/perf-prep.sh")
 323                 shell("./init-tools.sh")
 324                 copyArtifacts(fullBuildJobName) {
 325                     includePatterns("bin/**")
 326                     buildSelector {
 327                         buildNumber('\${PRODUCT_BUILD}')
 328                     }
 329                 }
 330                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 331                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 332                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 333                 shell("""./tests/scripts/run-xunit-perf.sh \\
 334                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 335                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 336                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 337                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 338                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 339                 --runType=\"${runType}\" \\
 340                 --benchViewOS=\"${os}\" \\
 341                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 342                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 343                 --uploadToBenchview""")
 344                 shell("mkdir -p bin/toArchive/sandbox/Logs/")
 345                 shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
 346             }
 347         }
 348
 349         def archiveSettings = new ArchivalSettings()
 350         archiveSettings.addFiles('bin/toArchive/**')
 351         archiveSettings.addFiles('machinedata.json')
 352
 353         Utilities.addArchival(newJob, archiveSettings)
 354         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 355
 356         // For perf, we need to keep the run results longer
 357         newJob.with {
 358             // Enable the log rotator
 359             logRotator {
 360                 artifactDaysToKeep(30)
 361                 daysToKeep(30)
 362                 artifactNumToKeep(200)
 363                 numToKeep(200)
 364             }
 365             wrappers {
 366                 timeout {
 367                     absolute(240)
 368                 }
 369             }
 370         }
 371     } // os
 372
 373     def flowJobPerfRunList = perfOSList.collect { os ->
 374         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 375     }
 376     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 377         if (isPR) {
 378             parameters {
 379                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 380             }
 381         }
 382         buildFlow("""
 383 // First, build the bits on RHEL7.2
 384 b = build(params, '${fullBuildJobName}')
 385
 386 // Then, run the perf tests
 387 parallel(
 388     ${flowJobPerfRunList.join(",\n    ")}
 389 )
 390 """)
 391     }
 392
 393     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 394     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 395
 396     if (isPR) {
 397         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 398         builder.setGithubContext("Linux Perf Test Flow")
 399         builder.triggerOnlyOnComment()
 400         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 401         builder.triggerForBranch(branch)
 402         builder.emitTrigger(newFlowJob)
 403     }
 404     else {
 405         // Set a push trigger
 406         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 407         builder.emitTrigger(newFlowJob)
 408     }
 409
 410 } // isPR
 411
 412 def static getFullThroughputJobName(def project, def os, def isPR) {
 413     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 414 }
 415
 416 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 417 [true, false].each { isPR ->
 418     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 419     def architecture = 'x64'
 420     def configuration = 'Release'
 421
 422     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 423     ['RHEL7.2'].each { os ->
 424         def newBuildJob = job(fullBuildJobName) {
 425             steps {
 426                 shell("./build.sh verbose ${architecture} ${configuration}")
 427             }
 428         }
 429         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 430         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 431         Utilities.addArchival(newBuildJob, "bin/Product/**")
 432     }
 433
 434     // Actual perf testing on the following OSes
 435     def throughputOSList = ['Ubuntu14.04']
 436     def throughputOptLevelList = ['full_opt', 'min_opt']
 437
 438     def throughputOSOptLevelList = []
 439
 440     throughputOSList.each { os ->
 441         throughputOptLevelList.each { opt_level ->
 442             throughputOSOptLevelList.add("${os}_${opt_level}")
 443         }
 444     }
 445
 446     throughputOSList.each { os ->
 447         throughputOptLevelList.each { opt_level ->
 448             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 449
 450                 label('linux_clr_perf')
 451                     wrappers {
 452                         credentialsBinding {
 453                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 454                         }
 455                     }
 456
 457                 if (isPR) {
 458                     parameters {
 459                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 460                     }
 461                 }
 462
 463                 parameters {
 464                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 465                 }
 466
 467                 def osGroup = getOSGroup(os)
 468                 def runType = isPR ? 'private' : 'rolling'
 469                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 470
 471                 steps {
 472                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 473                     shell("./init-tools.sh")
 474                     copyArtifacts(fullBuildJobName) {
 475                         includePatterns("bin/Product/**")
 476                         buildSelector {
 477                             buildNumber('\${PRODUCT_BUILD}')
 478                         }
 479                     }
 480                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 481                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 482                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 483                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 484                     -arch \"${architecture}\" \\
 485                     -os \"${os}\" \\
 486                     -configuration \"${configuration}\" \\
 487                     -opt_level \"${opt_level}\" \\
 488                     -clr_root \"\${WORKSPACE}\" \\
 489                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 490                     -run_type \"${runType}\" \\
 491                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 492                 }
 493             }
 494
 495             // Save machinedata.json to /artifact/bin/ Jenkins dir
 496             def archiveSettings = new ArchivalSettings()
 497             archiveSettings.addFiles('throughput-*.csv')
 498             archiveSettings.addFiles('machinedata.json')
 499             Utilities.addArchival(newJob, archiveSettings)
 500
 501             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 502
 503             // For perf, we need to keep the run results longer
 504             newJob.with {
 505                 // Enable the log rotator
 506                 logRotator {
 507                     artifactDaysToKeep(7)
 508                     daysToKeep(300)
 509                     artifactNumToKeep(25)
 510                     numToKeep(1000)
 511                 }
 512             }
 513         } // opt_level
 514     } // os
 515
 516     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 517         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 518     }
 519     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 520         if (isPR) {
 521             parameters {
 522                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 523             }
 524         }
 525         buildFlow("""
 526 // First, build the bits on RHEL7.2
 527 b = build(params, '${fullBuildJobName}')
 528
 529 // Then, run the perf tests
 530 parallel(
 531     ${flowJobTPRunList.join(",\n    ")}
 532 )
 533 """)
 534     }
 535
 536     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 537     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 538
 539     if (isPR) {
 540         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 541         builder.setGithubContext("Linux Throughput Perf Test Flow")
 542         builder.triggerOnlyOnComment()
 543         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 544         builder.triggerForBranch(branch)
 545         builder.emitTrigger(newFlowJob)
 546     }
 547     else {
 548         // Set a push trigger
 549         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 550         builder.emitTrigger(newFlowJob)
 551     }
 552
 553 } // isPR
 554
 555 // Setup CoreCLR-Scenarios tests
 556 [true, false].each { isPR ->
 557     ['Windows_NT'].each { os ->
 558         ['x64', 'x86'].each { arch ->
 559             ['ryujit'].each { jit ->
 560                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 561                     def architecture = arch
 562                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 563
 564                         def testEnv = ""
 565
 566                         // Set the label.
 567                         label('windows_server_2016_clr_perf')
 568                         wrappers {
 569                             credentialsBinding {
 570                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 571                             }
 572                         }
 573
 574                         if (isPR) {
 575                             parameters {
 576                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 577                             }
 578                         }
 579
 580                         parameters {
 581                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 582                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 583                         }
 584
 585                         def configuration = 'Release'
 586                         def runType = isPR ? 'private' : 'rolling'
 587                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 588                         def uploadString = '-uploadToBenchview'
 589
 590                         steps {
 591                             // Batch
 592                             batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 593                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 594                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 595
 596                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 597                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 598                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 599                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 600                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 601                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 602                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 603                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 604                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 605
 606                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 607
 608                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
 609
 610                             // Profile=Off
 611                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 612                             batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\Off\\")
 613
 614                             // Profile=On
 615                             if (opt_level != 'min_opt') {
 616                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 617                                 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\On\\")
 618                             }
 619                         }
 620                     }
 621
 622                     def archiveSettings = new ArchivalSettings()
 623                     archiveSettings.addFiles('bin/toArchive/**')
 624                     archiveSettings.addFiles('machinedata.json')
 625
 626                     Utilities.addArchival(newJob, archiveSettings)
 627                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 628
 629                     newJob.with {
 630                         logRotator {
 631                             artifactDaysToKeep(30)
 632                             daysToKeep(30)
 633                             artifactNumToKeep(200)
 634                             numToKeep(200)
 635                         }
 636                         wrappers {
 637                             timeout {
 638                                 absolute(240)
 639                             }
 640                         }
 641                     }
 642
 643                     if (isPR) {
 644                         def opts = ""
 645                         if (opt_level == 'min_opt') {
 646                             opts = '\\W+min_opts'
 647                         }
 648                         def jitt = ""
 649                         if (jit != 'ryujit') {
 650                             jitt = "\\W+${jit}"
 651                         }
 652
 653                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 654                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 655                         builder.triggerOnlyOnComment()
 656                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 657                         builder.triggerForBranch(branch)
 658                         builder.emitTrigger(newJob)
 659                     }
 660                     else {
 661                         // Set a push trigger
 662                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 663                         builder.emitTrigger(newJob)
 664                     }
 665                 }
 666             }
 667         }
 668     }
 669 }
 670
 671 // Setup size-on-disk test
 672 ['Windows_NT'].each { os ->
 673     ['x64', 'x86'].each { arch ->
 674         def architecture = arch
 675         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 676
 677             wrappers {
 678                 credentialsBinding {
 679                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 680                 }
 681             }
 682
 683             def channel = 'master'
 684             def configuration = 'Release'
 685             def runType = 'rolling'
 686             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 687             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 688             def coreRoot = "${testBin}\\Tests\\Core_Root"
 689             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 690
 691             steps {
 692                 // Install nuget and get BenchView tools
 693                 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 694                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 695                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 696
 697                 // Generate submission metadata for BenchView
 698                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 699                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 700                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 701                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 702                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 703                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 704                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 705
 706                 // Generate machine data from BenchView
 707                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 708
 709                 // Build CoreCLR and gnerate test layout
 710                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 711                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 712
 713                 // Run the size on disk benchmark
 714                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 715
 716                 // From sodbench.csv, create measurment.json, then submission.json
 717                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 718                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 719
 720                 // If this is a PR, upload submission.json
 721                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 722             }
 723         }
 724
 725         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 726
 727         def archiveSettings = new ArchivalSettings()
 728         archiveSettings.addFiles('bin/toArchive/**')
 729         archiveSettings.addFiles('machinedata.json')
 730
 731         Utilities.addArchival(newJob, archiveSettings)
 732         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 733
 734         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 735         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 736
 737         newJob.with {
 738             logRotator {
 739                 artifactDaysToKeep(30)
 740                 daysToKeep(30)
 741                 artifactNumToKeep(200)
 742                 numToKeep(200)
 743             }
 744             wrappers {
 745                 timeout {
 746                     absolute(240)
 747                 }
 748             }
 749         }
 750     }
 751 }
 752
 753 // Setup IlLink tests
 754 [true, false].each { isPR ->
 755     ['Windows_NT'].each { os ->
 756         ['x64'].each { arch ->
 757             ['ryujit'].each { jit ->
 758                 ['full_opt'].each { opt_level ->
 759                     def architecture = arch
 760                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 761
 762                         def testEnv = ""
 763                         wrappers {
 764                             credentialsBinding {
 765                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 766                             }
 767                         }
 768
 769                         if (isPR) {
 770                             parameters {
 771                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 772                             }
 773                         }
 774
 775                         parameters {
 776                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 777                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 778                         }
 779
 780                         def configuration = 'Release'
 781                         def runType = isPR ? 'private' : 'rolling'
 782                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 783                         def uploadString = '-uploadToBenchview'
 784
 785                         steps {
 786                             // Batch
 787                             batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 788                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 789                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 790
 791                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 792                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 793                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 794                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 795                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 796                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 797                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 798                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 799                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 800
 801                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 802
 803                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
 804
 805                             // Scenario: ILLink
 806                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 807                             batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
 808                         }
 809                     }
 810
 811                     def archiveSettings = new ArchivalSettings()
 812                     archiveSettings.addFiles('bin/toArchive/**')
 813                     archiveSettings.addFiles('machinedata.json')
 814
 815                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 816                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 817                     Utilities.addArchival(newJob, archiveSettings)
 818                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 819
 820                     newJob.with {
 821                         logRotator {
 822                             artifactDaysToKeep(30)
 823                             daysToKeep(30)
 824                             artifactNumToKeep(200)
 825                             numToKeep(200)
 826                         }
 827                         wrappers {
 828                             timeout {
 829                                 absolute(240)
 830                             }
 831                         }
 832                     }
 833
 834                     if (isPR) {
 835                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 836                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 837                         builder.triggerOnlyOnComment()
 838                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 839                         builder.triggerForBranch(branch)
 840                         builder.emitTrigger(newJob)
 841                     }
 842                     else {
 843                         // Set a push trigger
 844                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 845                         builder.emitTrigger(newJob)
 846                     }
 847                 }
 848             }
 849         }
 850     }
 851 }
 852
 853 Utilities.createHelperJob(this, project, branch,
 854     "Welcome to the ${project} Perf help",
 855     "Have a nice day!")