perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 ['ryujit', 'legacy_backend'].each { jit ->
  34
  35                     if (arch == 'x64' && jit == 'legacy_backend') {
  36                         return
  37                     }
  38
  39                     ['full_opt', 'min_opt'].each { opt_level ->
  40                         if (isSmoketest && opt_level == 'min_opt') {
  41                             return
  42                         }
  43
  44                         def architecture = arch
  45                         def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  46                         def testEnv = ""
  47
  48                         if (jit == 'legacy_backend') {
  49                             testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
  50                         }
  51
  52                         def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  53                             // Set the label.
  54                             label('windows_server_2016_clr_perf')
  55                             wrappers {
  56                                 credentialsBinding {
  57                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  58                                 }
  59                             }
  60
  61                             if (isPR) {
  62                                 parameters {
  63                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  64                                 }
  65                             }
  66                             if (isSmoketest) {
  67                                 parameters {
  68                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  69                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  70                                 }
  71                             }
  72                             else {
  73                                 parameters {
  74                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  75                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  76                                 }
  77                             }
  78
  79                             def configuration = 'Release'
  80                             def runType = isPR ? 'private' : 'rolling'
  81                             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  82                             def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  83
  84                             steps {
  85                                 // Batch
  86
  87                                 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  88                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  89                                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  90                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  91                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  92                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  93                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  94                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
  95                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
  96                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  97                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  98                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  99
 100                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 101
 102                                 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
 103
 104                                 // Run with just stopwatch: Profile=Off
 105                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
 106                                 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\Off\\")
 107
 108                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
 109                                 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\")
 110
 111                                 // Run with the full set of counters enabled: Profile=On
 112                                 if (opt_level != 'min_opt') {
 113                                     batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 114                                     batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
 115
 116                                     batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 117                                     batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
 118                                 }
 119                             }
 120                         }
 121
 122                         if (isSmoketest) {
 123                             Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 124                         }
 125
 126                         def archiveSettings = new ArchivalSettings()
 127                         archiveSettings.addFiles('bin/toArchive/**')
 128                         archiveSettings.addFiles('machinedata.json')
 129
 130                         Utilities.addArchival(newJob, archiveSettings)
 131                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 132
 133                         newJob.with {
 134                             logRotator {
 135                                 artifactDaysToKeep(30)
 136                                 daysToKeep(30)
 137                                 artifactNumToKeep(200)
 138                                 numToKeep(200)
 139                             }
 140                             wrappers {
 141                                 timeout {
 142                                     absolute(240)
 143                                 }
 144                             }
 145                         }
 146
 147                         if (isPR) {
 148                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 149                             if (isSmoketest) {
 150                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
 151                             }
 152                             else {
 153                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 154
 155                                 def opts = ""
 156                                 if (opt_level == 'min_opt') {
 157                                     opts = '\\W+min_opts'
 158                                 }
 159                                 def jitt = ""
 160                                 if (jit != 'ryujit') {
 161                                     jitt = "\\W+${jit}"
 162                                 }
 163
 164                                 builder.triggerOnlyOnComment()
 165                                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 166                             }
 167                             builder.triggerForBranch(branch)
 168                             builder.emitTrigger(newJob)
 169                         }
 170                         else {
 171                             // Set a push trigger
 172                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 173                             builder.emitTrigger(newJob)
 174                         }
 175                     }
 176                 }
 177             }
 178         }
 179     }
 180 }
 181
 182 // Setup throughput perflab tests runs
 183 [true, false].each { isPR ->
 184     ['Windows_NT'].each { os ->
 185         ['x64', 'x86'].each { arch ->
 186             ['ryujit', 'legacy_backend'].each { jit ->
 187                 [true, false].each { pgo_optimized ->
 188                     if (arch == 'x64' && jit == 'legacy_backend') {
 189                         return
 190                     }
 191
 192                     // pgo not supported for legacy_backend
 193                     if (pgo_optimized && jit == 'legacy_backend') {
 194                         return
 195                     }
 196
 197                     ['full_opt', 'min_opt'].each { opt_level ->
 198                         def architecture = arch
 199
 200                         pgo_build = ""
 201                         pgo_test = ""
 202                         pgo_string = "pgo"
 203                         if (!pgo_optimized) {
 204                             pgo_build = " -nopgooptimize"
 205                             pgo_test = " -nopgo"
 206                             pgo_string = "nopgo"
 207                         }
 208
 209                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 210                             // Set the label.
 211                             label('windows_server_2016_clr_perf')
 212                             wrappers {
 213                                 credentialsBinding {
 214                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 215                                 }
 216                             }
 217
 218                             if (isPR) {
 219                                 parameters {
 220                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 221                                 }
 222                             }
 223
 224                             def configuration = 'Release'
 225                             def runType = isPR ? 'private' : 'rolling'
 226                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 227
 228                             steps {
 229                                 // Batch
 230                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 231                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 232                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 233                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 234                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 235                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 236                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 237                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 238                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 239                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 240                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 241                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 242                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 243                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 244                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 245                             }
 246                         }
 247
 248                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 249                         def archiveSettings = new ArchivalSettings()
 250                         archiveSettings.addFiles('throughput-*.csv')
 251                         Utilities.addArchival(newJob, archiveSettings)
 252
 253                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 254
 255                         if (isPR) {
 256                             def opts = ""
 257                             if (opt_level == 'min_opt') {
 258                                 opts = '\\W+min_opts'
 259                             }
 260
 261                             def jitt = ""
 262                             if (jit != 'ryujit') {
 263                                 jitt = "\\W+${jit}"
 264                             }
 265
 266                             def pgo_trigger = ""
 267                             if (pgo_optimized) {
 268                                 pgo_trigger = "\\W+nopgo"
 269                             }
 270
 271
 272                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 273                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 274                             builder.triggerOnlyOnComment()
 275                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 276                             builder.triggerForBranch(branch)
 277                             builder.emitTrigger(newJob)
 278                         }
 279                         else {
 280                             // Set a push trigger
 281                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 282                             builder.emitTrigger(newJob)
 283                         }
 284                     }
 285                 }
 286             }
 287         }
 288     }
 289 }
 290
 291 def static getFullPerfJobName(def project, def os, def isPR) {
 292     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 293 }
 294
 295 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 296 [true, false].each { isPR ->
 297     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 298     def architecture = 'x64'
 299     def configuration = 'Release'
 300
 301     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 302     ['RHEL7.2'].each { os ->
 303         def newBuildJob = job(fullBuildJobName) {
 304             steps {
 305                 shell("./build.sh verbose ${architecture} ${configuration}")
 306             }
 307         }
 308         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 309         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 310         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 311     }
 312
 313
 314     // Actual perf testing on the following OSes
 315     def perfOSList = ['Ubuntu14.04']
 316     perfOSList.each { os ->
 317         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 318
 319             label('linux_clr_perf')
 320             wrappers {
 321                 credentialsBinding {
 322                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 323                 }
 324             }
 325
 326             if (isPR) {
 327                 parameters {
 328                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 329                 }
 330             }
 331
 332             parameters {
 333                 // Cap the maximum number of iterations to 21.
 334                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 335                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 336                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 337             }
 338
 339             def osGroup = getOSGroup(os)
 340             def runType = isPR ? 'private' : 'rolling'
 341             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 342
 343             steps {
 344                 shell("./tests/scripts/perf-prep.sh")
 345                 shell("./init-tools.sh")
 346                 copyArtifacts(fullBuildJobName) {
 347                     includePatterns("bin/**")
 348                     buildSelector {
 349                         buildNumber('\${PRODUCT_BUILD}')
 350                     }
 351                 }
 352                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 353                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 354                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 355                 shell("""./tests/scripts/run-xunit-perf.sh \\
 356                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 357                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 358                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 359                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 360                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 361                 --runType=\"${runType}\" \\
 362                 --benchViewOS=\"${os}\" \\
 363                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 364                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 365                 --uploadToBenchview""")
 366                 shell("mkdir -p bin/toArchive/sandbox/Logs/")
 367                 shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
 368             }
 369         }
 370
 371         def archiveSettings = new ArchivalSettings()
 372         archiveSettings.addFiles('bin/toArchive/**')
 373         archiveSettings.addFiles('machinedata.json')
 374
 375         Utilities.addArchival(newJob, archiveSettings)
 376         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 377
 378         // For perf, we need to keep the run results longer
 379         newJob.with {
 380             // Enable the log rotator
 381             logRotator {
 382                 artifactDaysToKeep(30)
 383                 daysToKeep(30)
 384                 artifactNumToKeep(200)
 385                 numToKeep(200)
 386             }
 387             wrappers {
 388                 timeout {
 389                     absolute(240)
 390                 }
 391             }
 392         }
 393     } // os
 394
 395     def flowJobPerfRunList = perfOSList.collect { os ->
 396         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 397     }
 398     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 399         if (isPR) {
 400             parameters {
 401                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 402             }
 403         }
 404         buildFlow("""
 405 // First, build the bits on RHEL7.2
 406 b = build(params, '${fullBuildJobName}')
 407
 408 // Then, run the perf tests
 409 parallel(
 410     ${flowJobPerfRunList.join(",\n    ")}
 411 )
 412 """)
 413     }
 414
 415     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 416     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 417
 418     if (isPR) {
 419         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 420         builder.setGithubContext("Linux Perf Test Flow")
 421         builder.triggerOnlyOnComment()
 422         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 423         builder.triggerForBranch(branch)
 424         builder.emitTrigger(newFlowJob)
 425     }
 426     else {
 427         // Set a push trigger
 428         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 429         builder.emitTrigger(newFlowJob)
 430     }
 431
 432 } // isPR
 433
 434 def static getFullThroughputJobName(def project, def os, def isPR) {
 435     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 436 }
 437
 438 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 439 [true, false].each { isPR ->
 440     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 441     def architecture = 'x64'
 442     def configuration = 'Release'
 443
 444     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 445     ['RHEL7.2'].each { os ->
 446         def newBuildJob = job(fullBuildJobName) {
 447             steps {
 448                 shell("./build.sh verbose ${architecture} ${configuration}")
 449             }
 450         }
 451         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 452         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 453         Utilities.addArchival(newBuildJob, "bin/Product/**")
 454     }
 455
 456     // Actual perf testing on the following OSes
 457     def throughputOSList = ['Ubuntu14.04']
 458     def throughputOptLevelList = ['full_opt', 'min_opt']
 459
 460     def throughputOSOptLevelList = []
 461
 462     throughputOSList.each { os ->
 463         throughputOptLevelList.each { opt_level ->
 464             throughputOSOptLevelList.add("${os}_${opt_level}")
 465         }
 466     }
 467
 468     throughputOSList.each { os ->
 469         throughputOptLevelList.each { opt_level ->
 470             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 471
 472                 label('linux_clr_perf')
 473                     wrappers {
 474                         credentialsBinding {
 475                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 476                         }
 477                     }
 478
 479                 if (isPR) {
 480                     parameters {
 481                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 482                     }
 483                 }
 484
 485                 parameters {
 486                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 487                 }
 488
 489                 def osGroup = getOSGroup(os)
 490                 def runType = isPR ? 'private' : 'rolling'
 491                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 492
 493                 steps {
 494                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 495                     shell("./init-tools.sh")
 496                     copyArtifacts(fullBuildJobName) {
 497                         includePatterns("bin/Product/**")
 498                         buildSelector {
 499                             buildNumber('\${PRODUCT_BUILD}')
 500                         }
 501                     }
 502                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 503                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 504                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 505                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 506                     -arch \"${architecture}\" \\
 507                     -os \"${os}\" \\
 508                     -configuration \"${configuration}\" \\
 509                     -opt_level \"${opt_level}\" \\
 510                     -clr_root \"\${WORKSPACE}\" \\
 511                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 512                     -run_type \"${runType}\" \\
 513                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 514                 }
 515             }
 516
 517             // Save machinedata.json to /artifact/bin/ Jenkins dir
 518             def archiveSettings = new ArchivalSettings()
 519             archiveSettings.addFiles('throughput-*.csv')
 520             archiveSettings.addFiles('machinedata.json')
 521             Utilities.addArchival(newJob, archiveSettings)
 522
 523             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 524
 525             // For perf, we need to keep the run results longer
 526             newJob.with {
 527                 // Enable the log rotator
 528                 logRotator {
 529                     artifactDaysToKeep(7)
 530                     daysToKeep(300)
 531                     artifactNumToKeep(25)
 532                     numToKeep(1000)
 533                 }
 534             }
 535         } // opt_level
 536     } // os
 537
 538     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 539         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 540     }
 541     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 542         if (isPR) {
 543             parameters {
 544                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 545             }
 546         }
 547         buildFlow("""
 548 // First, build the bits on RHEL7.2
 549 b = build(params, '${fullBuildJobName}')
 550
 551 // Then, run the perf tests
 552 parallel(
 553     ${flowJobTPRunList.join(",\n    ")}
 554 )
 555 """)
 556     }
 557
 558     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 559     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 560
 561     if (isPR) {
 562         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 563         builder.setGithubContext("Linux Throughput Perf Test Flow")
 564         builder.triggerOnlyOnComment()
 565         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 566         builder.triggerForBranch(branch)
 567         builder.emitTrigger(newFlowJob)
 568     }
 569     else {
 570         // Set a push trigger
 571         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 572         builder.emitTrigger(newFlowJob)
 573     }
 574
 575 } // isPR
 576
 577 // Setup CoreCLR-Scenarios tests
 578 [true, false].each { isPR ->
 579     ['Windows_NT'].each { os ->
 580         ['x64', 'x86'].each { arch ->
 581             ['ryujit', 'legacy_backend'].each { jit ->
 582
 583                 if (arch == 'x64' && jit == 'legacy_backend') {
 584                     return
 585                 }
 586
 587                 ['full_opt', 'min_opt'].each { opt_level ->
 588                     def architecture = arch
 589                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 590
 591                         def testEnv = ""
 592                         if (jit == 'legacy_backend') {
 593                             testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
 594                         }
 595
 596                         // Set the label.
 597                         label('windows_server_2016_clr_perf')
 598                         wrappers {
 599                             credentialsBinding {
 600                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 601                             }
 602                         }
 603
 604                         if (isPR) {
 605                             parameters {
 606                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 607                             }
 608                         }
 609
 610                         parameters {
 611                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 612                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 613                         }
 614
 615                         def configuration = 'Release'
 616                         def runType = isPR ? 'private' : 'rolling'
 617                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 618                         def uploadString = '-uploadToBenchview'
 619
 620                         steps {
 621                             // Batch
 622                             batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 623                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 624                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 625
 626                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 627                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 628                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 629                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 630                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 631                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
 632                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 633                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 634                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 635
 636                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 637
 638                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
 639
 640                             // Scenario: JitBench
 641                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 642                             batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\")
 643                         }
 644                     }
 645
 646                     def archiveSettings = new ArchivalSettings()
 647                     archiveSettings.addFiles('bin/toArchive/**')
 648                     archiveSettings.addFiles('machinedata.json')
 649
 650                     Utilities.addArchival(newJob, archiveSettings)
 651                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 652
 653                     newJob.with {
 654                         logRotator {
 655                             artifactDaysToKeep(30)
 656                             daysToKeep(30)
 657                             artifactNumToKeep(200)
 658                             numToKeep(200)
 659                         }
 660                         wrappers {
 661                             timeout {
 662                                 absolute(240)
 663                             }
 664                         }
 665                     }
 666
 667                     if (isPR) {
 668                         def opts = ""
 669                         if (opt_level == 'min_opt') {
 670                             opts = '\\W+min_opts'
 671                         }
 672                         def jitt = ""
 673                         if (jit != 'ryujit') {
 674                             jitt = "\\W+${jit}"
 675                         }
 676
 677                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 678                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 679                         builder.triggerOnlyOnComment()
 680                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}{$opts}${jitt}\\W+perf\\W+scenarios.*")
 681                         builder.triggerForBranch(branch)
 682                         builder.emitTrigger(newJob)
 683                     }
 684                     else {
 685                         // Set a push trigger
 686                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 687                         builder.emitTrigger(newJob)
 688                     }
 689                 }
 690             }
 691         }
 692     }
 693 }
 694
 695 // Setup size-on-disk test
 696 ['Windows_NT'].each { os ->
 697     ['x64', 'x86'].each { arch ->
 698         def architecture = arch
 699         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 700
 701             wrappers {
 702                 credentialsBinding {
 703                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 704                 }
 705             }
 706
 707             def channel = 'master'
 708             def configuration = 'Release'
 709             def runType = 'rolling'
 710             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 711             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 712             def coreRoot = "${testBin}\\Tests\\Core_Root"
 713             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 714
 715             steps {
 716                 // Install nuget and get BenchView tools
 717                 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 718                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 719                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 720
 721                 // Generate submission metadata for BenchView
 722                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 723                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 724                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 725                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 726                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 727                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
 728                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 729
 730                 // Generate machine data from BenchView
 731                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 732
 733                 // Build CoreCLR and gnerate test layout
 734                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 735                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 736
 737                 // Run the size on disk benchmark
 738                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 739
 740                 // From sodbench.csv, create measurment.json, then submission.json
 741                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 742                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 743
 744                 // If this is a PR, upload submission.json
 745                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 746             }
 747         }
 748
 749         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 750
 751         def archiveSettings = new ArchivalSettings()
 752         archiveSettings.addFiles('bin/toArchive/**')
 753         archiveSettings.addFiles('machinedata.json')
 754
 755         Utilities.addArchival(newJob, archiveSettings)
 756         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 757
 758         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 759         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 760
 761         newJob.with {
 762             logRotator {
 763                 artifactDaysToKeep(30)
 764                 daysToKeep(30)
 765                 artifactNumToKeep(200)
 766                 numToKeep(200)
 767             }
 768             wrappers {
 769                 timeout {
 770                     absolute(240)
 771                 }
 772             }
 773         }
 774     }
 775 }
 776
 777 // Setup IlLink tests
 778 [true, false].each { isPR ->
 779     ['Windows_NT'].each { os ->
 780         ['x64'].each { arch ->
 781             ['ryujit'].each { jit ->
 782                 ['full_opt'].each { opt_level ->
 783                     def architecture = arch
 784                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 785
 786                         def testEnv = ""
 787                         wrappers {
 788                             credentialsBinding {
 789                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 790                             }
 791                         }
 792
 793                         if (isPR) {
 794                             parameters {
 795                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 796                             }
 797                         }
 798
 799                         parameters {
 800                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 801                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 802                         }
 803
 804                         def configuration = 'Release'
 805                         def runType = isPR ? 'private' : 'rolling'
 806                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 807                         def uploadString = '-uploadToBenchview'
 808
 809                         steps {
 810                             // Batch
 811                             batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 812                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 813                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 814
 815                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 816                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 817                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 818                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 819                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 820                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
 821                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 822                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 823                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 824
 825                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 826
 827                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
 828
 829                             // Scenario: ILLink
 830                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 831                             batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
 832                         }
 833                     }
 834
 835                     def archiveSettings = new ArchivalSettings()
 836                     archiveSettings.addFiles('bin/toArchive/**')
 837                     archiveSettings.addFiles('machinedata.json')
 838
 839                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 840                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 841                     Utilities.addArchival(newJob, archiveSettings)
 842                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 843
 844                     newJob.with {
 845                         logRotator {
 846                             artifactDaysToKeep(30)
 847                             daysToKeep(30)
 848                             artifactNumToKeep(200)
 849                             numToKeep(200)
 850                         }
 851                         wrappers {
 852                             timeout {
 853                                 absolute(240)
 854                             }
 855                         }
 856                     }
 857
 858                     if (isPR) {
 859                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 860                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 861                         builder.triggerOnlyOnComment()
 862                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 863                         builder.triggerForBranch(branch)
 864                         builder.emitTrigger(newJob)
 865                     }
 866                     else {
 867                         // Set a push trigger
 868                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 869                         builder.emitTrigger(newJob)
 870                     }
 871                 }
 872             }
 873         }
 874     }
 875 }
 876
 877 Utilities.createHelperJob(this, project, branch,
 878     "Welcome to the ${project} Perf help",
 879     "Have a nice day!")