perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 ['ryujit'].each { jit ->
  34                     ['full_opt', 'min_opt'].each { opt_level ->
  35
  36                         def architecture = arch
  37                         def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  38                         def testEnv = ""
  39
  40                         def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  41                             // Set the label.
  42                             label('windows_server_2016_clr_perf')
  43                             wrappers {
  44                                 credentialsBinding {
  45                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  46                                 }
  47                             }
  48
  49                             if (isPR) {
  50                                 parameters {
  51                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  52                                 }
  53                             }
  54
  55                             if (isSmoketest) {
  56                                 parameters {
  57                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  58                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  59                                 }
  60                             }
  61                             else{
  62                                 parameters {
  63                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  64                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  65                                 }
  66                             }
  67
  68                             def configuration = 'Release'
  69                             def runType = isPR ? 'private' : 'rolling'
  70                             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  71                             def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  72
  73                             steps {
  74                                 // Batch
  75
  76                                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  77                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  78                                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  79                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  80                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  81                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  82                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  83                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  84                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  85                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  86                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  87                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  88
  89                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  90
  91                                 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
  92
  93                                 // Run with just stopwatch: Profile=Off
  94                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  95                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  96
  97                                 // Run with the full set of counters enabled: Profile=On
  98                                 if (opt_level != 'min_opt') {
  99                                     batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 100                                     batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 101                                 }
 102                             }
 103                         }
 104
 105                         if (isSmoketest) {
 106                             Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 107                         }
 108                         def archiveSettings = new ArchivalSettings()
 109                         archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 110                         archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 111                         archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 112                         archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 113                         archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 114                         archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 115                         archiveSettings.addFiles('machinedata.json')
 116                         archiveSettings.setAlwaysArchive()
 117
 118                         Utilities.addArchival(newJob, archiveSettings)
 119                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 120
 121                         newJob.with {
 122                             logRotator {
 123                                 artifactDaysToKeep(14)
 124                                 daysToKeep(30)
 125                                 artifactNumToKeep(100)
 126                                 numToKeep(200)
 127                             }
 128                             wrappers {
 129                                 timeout {
 130                                     absolute(240)
 131                                 }
 132                             }
 133                         }
 134
 135                         if (isPR) {
 136                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 137                             if (isSmoketest) {
 138                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
 139                             }
 140                             else {
 141                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 142
 143                                 def opts = ""
 144                                 if (opt_level == 'min_opt') {
 145                                     opts = '\\W+min_opts'
 146                                 }
 147                                 def jitt = ""
 148                                 if (jit != 'ryujit') {
 149                                     jitt = "\\W+${jit}"
 150                                 }
 151
 152                                 builder.triggerOnlyOnComment()
 153                                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 154                             }
 155
 156                             builder.triggerForBranch(branch)
 157                             builder.emitTrigger(newJob)
 158                         }
 159                         else if (opt_level == 'full_opt') {
 160                             // Set a push trigger
 161                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 162                             builder.emitTrigger(newJob)
 163                         }
 164                         else {
 165                             // Set periodic trigger
 166                             Utilities.addPeriodicTrigger(newJob, '@daily')
 167                         }
 168                     }
 169                 }
 170             }
 171         }
 172     }
 173 }
 174
 175 // Setup throughput perflab tests runs
 176 [true, false].each { isPR ->
 177     ['Windows_NT'].each { os ->
 178         ['x64', 'x86'].each { arch ->
 179             ['ryujit'].each { jit ->
 180                 [true, false].each { pgo_optimized ->
 181                     ['full_opt', 'min_opt'].each { opt_level ->
 182                         def architecture = arch
 183
 184                         pgo_build = ""
 185                         pgo_test = ""
 186                         pgo_string = "pgo"
 187                         if (!pgo_optimized) {
 188                             pgo_build = " -nopgooptimize"
 189                             pgo_test = " -nopgo"
 190                             pgo_string = "nopgo"
 191                         }
 192
 193                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 194                             // Set the label.
 195                             label('windows_server_2016_clr_perf')
 196                             wrappers {
 197                                 credentialsBinding {
 198                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 199                                 }
 200                             }
 201
 202                             if (isPR) {
 203                                 parameters {
 204                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 205                                 }
 206                             }
 207
 208                             def configuration = 'Release'
 209                             def runType = isPR ? 'private' : 'rolling'
 210                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 211
 212                             steps {
 213                                 // Batch
 214                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 215                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 216                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 217                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 218                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 219                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 220                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 221                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 222                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 223                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 224                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 225                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 226                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 227                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 228                             }
 229                         }
 230
 231                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 232                         def archiveSettings = new ArchivalSettings()
 233                         archiveSettings.addFiles('throughput-*.csv')
 234                         archiveSettings.setAlwaysArchive()
 235                         Utilities.addArchival(newJob, archiveSettings)
 236
 237                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 238
 239                         if (isPR) {
 240                             def opts = ""
 241                             if (opt_level == 'min_opt') {
 242                                 opts = '\\W+min_opts'
 243                             }
 244
 245                             def jitt = ""
 246                             if (jit != 'ryujit') {
 247                                 jitt = "\\W+${jit}"
 248                             }
 249
 250                             def pgo_trigger = ""
 251                             if (pgo_optimized) {
 252                                 pgo_trigger = "\\W+nopgo"
 253                             }
 254
 255
 256                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 257                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 258                             builder.triggerOnlyOnComment()
 259                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 260                             builder.triggerForBranch(branch)
 261                             builder.emitTrigger(newJob)
 262                         }
 263                         else if (opt_level == 'full_opt' && pgo_optimized) {
 264                             // Set a push trigger
 265                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 266                             builder.emitTrigger(newJob)
 267                         }
 268                         else {
 269                             // Set periodic trigger
 270                             Utilities.addPeriodicTrigger(newJob, '@daily')
 271                         }
 272                     }
 273                 }
 274             }
 275         }
 276     }
 277 }
 278
 279 def static getFullPerfJobName(def project, def os, def arch, def isPR) {
 280     return Utilities.getFullJobName(project, "perf_${os}_${arch}", isPR)
 281 }
 282
 283 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 284 [true, false].each { isPR ->
 285     ['x64'].each { architecture ->
 286         def fullBuildJobName = Utilities.getFullJobName(project, "perf_linux_build", isPR)
 287         def configuration = 'Release'
 288
 289         def crossCompile = ""
 290         def crossLayout = ""
 291         def python = "python3.5"
 292
 293         // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 294         ['RHEL7.2'].each { os ->
 295             def newBuildJob = job(fullBuildJobName) {
 296                 steps {
 297                     shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
 298                     shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}${crossLayout}")
 299                 }
 300             }
 301             Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 302             Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 303             Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
 304         }
 305
 306
 307         // Actual perf testing on the following OSes
 308         def perfOSList = ['Ubuntu16.04']
 309
 310         perfOSList.each { os ->
 311             def newJob = job(getFullPerfJobName(project, os, architecture, isPR)) {
 312
 313                 def machineLabel = 'ubuntu_1604_clr_perf'
 314
 315                 label(machineLabel)
 316                 wrappers {
 317                     credentialsBinding {
 318                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 319                     }
 320                 }
 321
 322                 if (isPR) {
 323                     parameters {
 324                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 325                     }
 326                 }
 327
 328                 parameters {
 329                     // Cap the maximum number of iterations to 21.
 330                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 331                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 332                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 333                 }
 334
 335                 def osGroup = getOSGroup(os)
 336                 def runType = isPR ? 'private' : 'rolling'
 337                 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 338                 def uploadString = '-uploadToBenchview'
 339
 340                 def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
 341
 342                 steps {
 343                     shell("./tests/scripts/perf-prep.sh --nocorefx")
 344                     shell("./init-tools.sh")
 345                     copyArtifacts(fullBuildJobName) {
 346                         includePatterns("bin/**")
 347                         buildSelector {
 348                             buildNumber('\${PRODUCT_BUILD}')
 349                         }
 350                     }
 351                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 352                     "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 353                     "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 354                     shell("""${python} ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
 355                 }
 356             }
 357
 358             def archiveSettings = new ArchivalSettings()
 359             archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 360             archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 361             archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 362             archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 363             archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 364             archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 365             archiveSettings.addFiles('machinedata.json')
 366             archiveSettings.setAlwaysArchive()
 367
 368             Utilities.addArchival(newJob, archiveSettings)
 369             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 370
 371             // For perf, we need to keep the run results longer
 372             newJob.with {
 373                 // Enable the log rotator
 374                 logRotator {
 375                     artifactDaysToKeep(14)
 376                     daysToKeep(30)
 377                     artifactNumToKeep(100)
 378                     numToKeep(200)
 379                 }
 380                 wrappers {
 381                     timeout {
 382                         absolute(240)
 383                     }
 384                 }
 385             }
 386         } // os
 387
 388         def flowJobPerfRunList = perfOSList.collect { os ->
 389             "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, architecture, isPR)}') }"
 390         }
 391         def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_${architecture}_flow", isPR, '')) {
 392             if (isPR) {
 393                 parameters {
 394                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 395                 }
 396             }
 397             buildFlow("""
 398 // First, build the bits on RHEL7.2
 399 b = build(params, '${fullBuildJobName}')
 400
 401 // Then, run the perf tests
 402 parallel(
 403     ${flowJobPerfRunList.join(",\n    ")}
 404 )
 405 """)
 406         }
 407
 408         Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 409         Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 410
 411         if (isPR) {
 412             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 413             builder.setGithubContext("Linux Perf Test Flow")
 414             builder.triggerOnlyOnComment()
 415             builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 416             builder.triggerForBranch(branch)
 417             builder.emitTrigger(newFlowJob)
 418         }
 419         else {
 420             // Set a push trigger
 421             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 422             builder.emitTrigger(newFlowJob)
 423         }
 424     } // architecture
 425 } // isPR
 426
 427 def static getDockerImageName(def architecture, def os, def isBuild) {
 428     // We must change some docker private images to official later
 429     if (isBuild) {
 430         if (architecture == 'arm') {
 431             if (os == 'Ubuntu') {
 432                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
 433             }
 434         }
 435     }
 436     println("Unknown architecture to use docker: ${architecture} ${os}");
 437     assert false
 438 }
 439
 440 def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
 441     return Utilities.getFullJobName(project, "perf_throughput_${os}_${arch}", isPR)
 442 }
 443
 444 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 445 [true, false].each { isPR ->
 446     ['x64','arm'].each { architecture ->
 447         def fullBuildJobName = Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_build", isPR)
 448         def configuration = 'Release'
 449
 450
 451         def crossCompile = ""
 452         def python = "python3.5"
 453
 454         if (architecture == "arm") {
 455             python = "python3.6"
 456             def buildCommands = []
 457             def newBuildJob = job(fullBuildJobName) {
 458                 def additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
 459                 def dockerImage = getDockerImageName(architecture, 'Ubuntu', true)
 460                 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
 461
 462                 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh release ${architecture} cross crosscomponent"
 463
 464                 steps {
 465                     buildCommands.each { buildCommand ->
 466                         shell(buildCommand)
 467                     }
 468                 }
 469
 470                 publishers {
 471                     azureVMAgentPostBuildAction {
 472                         agentPostBuildAction('Delete agent after build execution (when idle).')
 473                     }
 474                 }
 475             }
 476             Utilities.setMachineAffinity(newBuildJob, "Ubuntu16.04", 'latest-or-auto')
 477             Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 478             Utilities.addArchival(newBuildJob, "bin/Product/**")
 479         }
 480         else {
 481             // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 482             ['RHEL7.2'].each { os ->
 483                 def newBuildJob = job(fullBuildJobName) {
 484                     steps {
 485                         shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
 486                     }
 487                 }
 488                 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 489                 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 490                 Utilities.addArchival(newBuildJob, "bin/Product/**")
 491             }
 492         }
 493
 494         // Actual perf testing on the following OSes
 495         def throughputOSList = ['Ubuntu16.04']
 496         if (architecture == 'arm') {
 497             throughputOSList = ['Ubuntu14.04']
 498         }
 499         def throughputOptLevelList = ['full_opt', 'min_opt']
 500
 501         def throughputOSOptLevelList = []
 502
 503         throughputOSList.each { os ->
 504             throughputOptLevelList.each { opt_level ->
 505                 throughputOSOptLevelList.add("${os}_${opt_level}")
 506             }
 507         }
 508
 509         throughputOSList.each { os ->
 510             throughputOptLevelList.each { opt_level ->
 511                 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", architecture, isPR)) {
 512
 513                     def machineLabel = 'ubuntu_1604_clr_perf'
 514                     if (architecture == 'arm') {
 515                         machineLabel = 'ubuntu_1404_clr_perf_arm'
 516                     }
 517
 518                     label(machineLabel)
 519                         wrappers {
 520                             credentialsBinding {
 521                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 522                             }
 523                         }
 524
 525                     if (isPR) {
 526                         parameters {
 527                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 528                         }
 529                     }
 530
 531                     parameters {
 532                         stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 533                     }
 534
 535                     def osGroup = getOSGroup(os)
 536                     def runType = isPR ? 'private' : 'rolling'
 537                     def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 538                     def archString = architecture == 'arm' ? ' --arch=arm' : ''
 539
 540                     steps {
 541                         shell("bash ./tests/scripts/perf-prep.sh --throughput${archString}")
 542                         copyArtifacts(fullBuildJobName) {
 543                             includePatterns("bin/Product/**")
 544                             buildSelector {
 545                                 buildNumber('\${PRODUCT_BUILD}')
 546                             }
 547                         }
 548                         shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 549                         "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 550                         "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 551                         shell("""${python} ./tests/scripts/run-throughput-perf.py \\
 552                         -arch \"${architecture}\" \\
 553                         -os \"${os}\" \\
 554                         -configuration \"${configuration}\" \\
 555                         -opt_level \"${opt_level}\" \\
 556                         -clr_root \"\${WORKSPACE}\" \\
 557                         -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/lib\" \\
 558                         -run_type \"${runType}\" \\
 559                         -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 560                     }
 561                 }
 562
 563                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 564                 def archiveSettings = new ArchivalSettings()
 565                 archiveSettings.addFiles('throughput-*.csv')
 566                 archiveSettings.addFiles('machinedata.json')
 567                 archiveSettings.setAlwaysArchive()
 568                 Utilities.addArchival(newJob, archiveSettings)
 569
 570                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 571
 572                 // For perf, we need to keep the run results longer
 573                 newJob.with {
 574                     // Enable the log rotator
 575                     logRotator {
 576                         artifactDaysToKeep(7)
 577                         daysToKeep(300)
 578                         artifactNumToKeep(25)
 579                         numToKeep(1000)
 580                     }
 581                 }
 582             } // opt_level
 583         } // os
 584
 585         def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 586             "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, architecture, isPR)}') }"
 587         }
 588         def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_flow", isPR, '')) {
 589             if (isPR) {
 590                 parameters {
 591                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 592                 }
 593             }
 594             buildFlow("""
 595     // First, build the bits on RHEL7.2
 596     b = build(params, '${fullBuildJobName}')
 597
 598     // Then, run the perf tests
 599     parallel(
 600         ${flowJobTPRunList.join(",\n    ")}
 601     )
 602     """)
 603         }
 604
 605         Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 606         Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 607
 608         if (isPR) {
 609             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 610             builder.setGithubContext("Linux ${architecture} Throughput Perf Test Flow")
 611             builder.triggerOnlyOnComment()
 612             builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 613             builder.triggerForBranch(branch)
 614             builder.emitTrigger(newFlowJob)
 615         }
 616         else {
 617             // Set a push trigger
 618             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 619             builder.emitTrigger(newFlowJob)
 620         }
 621     } // architecture
 622 } // isPR
 623
 624 // Setup CoreCLR-Scenarios tests
 625 [true, false].each { isPR ->
 626     ['Windows_NT'].each { os ->
 627         ['x64', 'x86'].each { arch ->
 628             ['ryujit'].each { jit ->
 629                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 630                     def architecture = arch
 631                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 632
 633                         def testEnv = ""
 634
 635                         // Set the label.
 636                         label('windows_server_2016_clr_perf')
 637                         wrappers {
 638                             credentialsBinding {
 639                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 640                             }
 641                         }
 642
 643                         if (isPR) {
 644                             parameters {
 645                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 646                             }
 647                         }
 648
 649                         parameters {
 650                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 651                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 652                         }
 653
 654                         def configuration = 'Release'
 655                         def runType = isPR ? 'private' : 'rolling'
 656                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 657                         def uploadString = '-uploadToBenchview'
 658
 659                         steps {
 660                             // Batch
 661                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 662                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 663                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 664
 665                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 666                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 667                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 668                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 669                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 670                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 671                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 672                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 673                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 674
 675                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 676
 677                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
 678
 679                             // Profile=Off
 680                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 681
 682                             // Profile=On
 683                             if (opt_level != 'min_opt') {
 684                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 685                             }
 686                         }
 687                     }
 688
 689                     def archiveSettings = new ArchivalSettings()
 690                     archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 691                     archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 692                     archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 693                     archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 694                     archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 695                     archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 696                     archiveSettings.addFiles('machinedata.json')
 697                     archiveSettings.setAlwaysArchive()
 698
 699                     Utilities.addArchival(newJob, archiveSettings)
 700                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 701
 702                     newJob.with {
 703                         logRotator {
 704                             artifactDaysToKeep(14)
 705                             daysToKeep(30)
 706                             artifactNumToKeep(100)
 707                             numToKeep(200)
 708                         }
 709                         wrappers {
 710                             timeout {
 711                                 absolute(240)
 712                             }
 713                         }
 714                     }
 715
 716                     if (isPR) {
 717                         def opts = ""
 718                         if (opt_level == 'min_opt') {
 719                             opts = '\\W+min_opts'
 720                         }
 721                         def jitt = ""
 722                         if (jit != 'ryujit') {
 723                             jitt = "\\W+${jit}"
 724                         }
 725
 726                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 727                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 728                         builder.triggerOnlyOnComment()
 729                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 730                         builder.triggerForBranch(branch)
 731                         builder.emitTrigger(newJob)
 732                     }
 733                     else if (opt_level == 'full_opt') {
 734                         // Set a push trigger
 735                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 736                         builder.emitTrigger(newJob)
 737                     }
 738                     else {
 739                         // Set periodic trigger
 740                         Utilities.addPeriodicTrigger(newJob, '@daily')
 741                     }
 742                 }
 743             }
 744         }
 745     }
 746 }
 747
 748 // Setup size-on-disk test
 749 ['Windows_NT'].each { os ->
 750     ['x64', 'x86'].each { arch ->
 751         def architecture = arch
 752         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 753
 754             wrappers {
 755                 credentialsBinding {
 756                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 757                 }
 758             }
 759
 760             def channel = 'master'
 761             def configuration = 'Release'
 762             def runType = 'rolling'
 763             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 764             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 765             def coreRoot = "${testBin}\\Tests\\Core_Root"
 766             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 767
 768             steps {
 769                 // Install nuget and get BenchView tools
 770                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 771                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 772                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 773
 774                 // Generate submission metadata for BenchView
 775                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 776                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 777                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 778                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 779                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 780                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 781                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 782
 783                 // Generate machine data from BenchView
 784                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 785
 786                 // Build CoreCLR and gnerate test layout
 787                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 788                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 789
 790                 // Run the size on disk benchmark
 791                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 792
 793                 // From sodbench.csv, create measurment.json, then submission.json
 794                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 795                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 796
 797                 // If this is a PR, upload submission.json
 798                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 799             }
 800         }
 801
 802         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 803
 804         def archiveSettings = new ArchivalSettings()
 805         archiveSettings.addFiles('bin/toArchive/**')
 806         archiveSettings.addFiles('machinedata.json')
 807         archiveSettings.setAlwaysArchive()
 808
 809         Utilities.addArchival(newJob, archiveSettings)
 810         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 811
 812         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 813         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 814
 815         newJob.with {
 816             logRotator {
 817                 artifactDaysToKeep(14)
 818                 daysToKeep(30)
 819                 artifactNumToKeep(100)
 820                 numToKeep(200)
 821             }
 822             wrappers {
 823                 timeout {
 824                     absolute(240)
 825                 }
 826             }
 827         }
 828     }
 829 }
 830
 831 // Setup IlLink tests
 832 [true, false].each { isPR ->
 833     ['Windows_NT'].each { os ->
 834         ['x64'].each { arch ->
 835             ['ryujit'].each { jit ->
 836                 ['full_opt'].each { opt_level ->
 837                     def architecture = arch
 838                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 839
 840                         def testEnv = ""
 841                         wrappers {
 842                             credentialsBinding {
 843                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 844                             }
 845                         }
 846
 847                         if (isPR) {
 848                             parameters {
 849                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 850                             }
 851                         }
 852
 853                         parameters {
 854                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 855                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 856                         }
 857
 858                         def configuration = 'Release'
 859                         def runType = isPR ? 'private' : 'rolling'
 860                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 861                         def uploadString = '-uploadToBenchview'
 862
 863                         steps {
 864                             // Batch
 865                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 866                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 867                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 868
 869                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 870                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 871                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 872                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 873                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 874                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 875                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 876                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 877                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 878
 879                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 880
 881                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 882
 883                             // Scenario: ILLink
 884                             batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
 885                             "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 886                         }
 887                     }
 888
 889                     def archiveSettings = new ArchivalSettings()
 890                     archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 891                     archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 892                     archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 893                     archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 894                     archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 895                     archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 896                     archiveSettings.addFiles('machinedata.json')
 897                     archiveSettings.setAlwaysArchive()
 898
 899                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 900                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 901                     Utilities.addArchival(newJob, archiveSettings)
 902                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 903
 904                     newJob.with {
 905                         logRotator {
 906                             artifactDaysToKeep(14)
 907                             daysToKeep(30)
 908                             artifactNumToKeep(100)
 909                             numToKeep(200)
 910                         }
 911                         wrappers {
 912                             timeout {
 913                                 absolute(240)
 914                             }
 915                         }
 916                     }
 917
 918                     if (isPR) {
 919                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 920                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 921                         builder.triggerOnlyOnComment()
 922                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 923                         builder.triggerForBranch(branch)
 924                         builder.emitTrigger(newJob)
 925                     }
 926                     else {
 927                         // Set a push trigger
 928                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 929                         builder.emitTrigger(newJob)
 930                     }
 931                 }
 932             }
 933         }
 934     }
 935 }
 936
 937 Utilities.createHelperJob(this, project, branch,
 938     "Welcome to the ${project} Perf help",
 939     "Have a nice day!")