perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 ['ryujit'].each { jit ->
  34                     ['full_opt', 'min_opt'].each { opt_level ->
  35
  36                         def architecture = arch
  37                         def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  38                         def testEnv = ""
  39
  40                         def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  41                             // Set the label.
  42                             if (isSmoketest) {
  43                                 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
  44                             }
  45                             else {
  46                                 label('windows_server_2016_clr_perf')
  47                             }
  48
  49                             wrappers {
  50                                 credentialsBinding {
  51                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  52                                 }
  53                             }
  54
  55                             if (isPR) {
  56                                 parameters {
  57                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  58                                 }
  59                             }
  60
  61                             if (isSmoketest) {
  62                                 parameters {
  63                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  64                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  65                                 }
  66                             }
  67                             else{
  68                                 parameters {
  69                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  70                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  71                                 }
  72                             }
  73
  74                             def configuration = 'Release'
  75                             def runType = isPR ? 'private' : 'rolling'
  76                             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  77                             def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  78
  79                             steps {
  80                                 // Batch
  81
  82                                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  83                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  84                                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  85                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  86                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  87                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  88                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  89                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  90                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  91                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  92                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  93                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  94
  95                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  96
  97                                 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
  98
  99                                 // Run with just stopwatch: Profile=Off
 100                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
 101                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
 102
 103                                 // Run with the full set of counters enabled: Profile=On
 104                                 if (opt_level != 'min_opt') {
 105                                     batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 106                                     batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 107                                 }
 108                             }
 109                         }
 110
 111                         def archiveSettings = new ArchivalSettings()
 112                         archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 113                         archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 114                         archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 115                         archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 116                         archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 117                         archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 118                         archiveSettings.addFiles('machinedata.json')
 119                         archiveSettings.setAlwaysArchive()
 120
 121                         Utilities.addArchival(newJob, archiveSettings)
 122                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 123
 124                         newJob.with {
 125                             logRotator {
 126                                 artifactDaysToKeep(14)
 127                                 daysToKeep(30)
 128                                 artifactNumToKeep(100)
 129                                 numToKeep(200)
 130                             }
 131                             wrappers {
 132                                 timeout {
 133                                     absolute(240)
 134                                 }
 135                             }
 136                         }
 137
 138                         if (isPR) {
 139                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 140                             if (isSmoketest) {
 141                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
 142                             }
 143                             else {
 144                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 145
 146                                 def opts = ""
 147                                 if (opt_level == 'min_opt') {
 148                                     opts = '\\W+min_opts'
 149                                 }
 150                                 def jitt = ""
 151                                 if (jit != 'ryujit') {
 152                                     jitt = "\\W+${jit}"
 153                                 }
 154
 155                                 builder.triggerOnlyOnComment()
 156                                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 157                             }
 158
 159                             builder.triggerForBranch(branch)
 160                             builder.emitTrigger(newJob)
 161                         }
 162                         else if (opt_level == 'full_opt') {
 163                             // Set a push trigger
 164                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 165                             builder.emitTrigger(newJob)
 166                         }
 167                         else {
 168                             // Set periodic trigger
 169                             Utilities.addPeriodicTrigger(newJob, '@daily')
 170                         }
 171                     }
 172                 }
 173             }
 174         }
 175     }
 176 }
 177
 178 // Setup throughput perflab tests runs
 179 [true, false].each { isPR ->
 180     ['Windows_NT'].each { os ->
 181         ['x64', 'x86'].each { arch ->
 182             ['ryujit'].each { jit ->
 183                 [true, false].each { pgo_optimized ->
 184                     ['full_opt', 'min_opt'].each { opt_level ->
 185                         def architecture = arch
 186
 187                         pgo_build = ""
 188                         pgo_test = ""
 189                         pgo_string = "pgo"
 190                         if (!pgo_optimized) {
 191                             pgo_build = " -nopgooptimize"
 192                             pgo_test = " -nopgo"
 193                             pgo_string = "nopgo"
 194                         }
 195
 196                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 197                             // Set the label.
 198                             label('windows_server_2016_clr_perf')
 199                             wrappers {
 200                                 credentialsBinding {
 201                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 202                                 }
 203                             }
 204
 205                             if (isPR) {
 206                                 parameters {
 207                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 208                                 }
 209                             }
 210
 211                             def configuration = 'Release'
 212                             def runType = isPR ? 'private' : 'rolling'
 213                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 214
 215                             steps {
 216                                 // Batch
 217                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 218                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 219                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 220                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 221                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 222                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 223                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 224                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 225                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 226                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 227                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 228                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 229                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 230                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 231                             }
 232                         }
 233
 234                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 235                         def archiveSettings = new ArchivalSettings()
 236                         archiveSettings.addFiles('throughput-*.csv')
 237                         archiveSettings.setAlwaysArchive()
 238                         Utilities.addArchival(newJob, archiveSettings)
 239
 240                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 241
 242                         if (isPR) {
 243                             def opts = ""
 244                             if (opt_level == 'min_opt') {
 245                                 opts = '\\W+min_opts'
 246                             }
 247
 248                             def jitt = ""
 249                             if (jit != 'ryujit') {
 250                                 jitt = "\\W+${jit}"
 251                             }
 252
 253                             def pgo_trigger = ""
 254                             if (pgo_optimized) {
 255                                 pgo_trigger = "\\W+nopgo"
 256                             }
 257
 258
 259                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 260                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 261                             builder.triggerOnlyOnComment()
 262                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 263                             builder.triggerForBranch(branch)
 264                             builder.emitTrigger(newJob)
 265                         }
 266                         else if (opt_level == 'full_opt' && pgo_optimized) {
 267                             // Set a push trigger
 268                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 269                             builder.emitTrigger(newJob)
 270                         }
 271                         else {
 272                             // Set periodic trigger
 273                             Utilities.addPeriodicTrigger(newJob, '@daily')
 274                         }
 275                     }
 276                 }
 277             }
 278         }
 279     }
 280 }
 281
 282 def static getFullPerfJobName(def project, def os, def arch, def isPR) {
 283     return Utilities.getFullJobName(project, "perf_${os}_${arch}", isPR)
 284 }
 285
 286 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 287 [true, false].each { isPR ->
 288     ['x64'].each { architecture ->
 289         def fullBuildJobName = Utilities.getFullJobName(project, "perf_linux_build", isPR)
 290         def configuration = 'Release'
 291
 292         def crossCompile = ""
 293         def crossLayout = ""
 294         def python = "python3.5"
 295
 296         // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 297         ['RHEL7.2'].each { os ->
 298             def newBuildJob = job(fullBuildJobName) {
 299                 steps {
 300                     shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
 301                     shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}${crossLayout}")
 302                 }
 303             }
 304             Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 305             Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 306             Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
 307         }
 308
 309
 310         // Actual perf testing on the following OSes
 311         def perfOSList = ['Ubuntu16.04']
 312
 313         perfOSList.each { os ->
 314             def newJob = job(getFullPerfJobName(project, os, architecture, isPR)) {
 315
 316                 def machineLabel = 'ubuntu_1604_clr_perf'
 317
 318                 label(machineLabel)
 319                 wrappers {
 320                     credentialsBinding {
 321                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 322                     }
 323                 }
 324
 325                 if (isPR) {
 326                     parameters {
 327                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 328                     }
 329                 }
 330
 331                 parameters {
 332                     // Cap the maximum number of iterations to 21.
 333                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 334                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 335                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 336                 }
 337
 338                 def osGroup = getOSGroup(os)
 339                 def runType = isPR ? 'private' : 'rolling'
 340                 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 341                 def uploadString = '-uploadToBenchview'
 342
 343                 def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
 344
 345                 steps {
 346                     shell("./tests/scripts/perf-prep.sh --nocorefx")
 347                     shell("./init-tools.sh")
 348                     copyArtifacts(fullBuildJobName) {
 349                         includePatterns("bin/**")
 350                         buildSelector {
 351                             buildNumber('\${PRODUCT_BUILD}')
 352                         }
 353                     }
 354                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 355                     "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 356                     "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 357                     shell("""${python} ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
 358                 }
 359             }
 360
 361             def archiveSettings = new ArchivalSettings()
 362             archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 363             archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 364             archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 365             archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 366             archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 367             archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 368             archiveSettings.addFiles('machinedata.json')
 369             archiveSettings.setAlwaysArchive()
 370
 371             Utilities.addArchival(newJob, archiveSettings)
 372             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 373
 374             // For perf, we need to keep the run results longer
 375             newJob.with {
 376                 // Enable the log rotator
 377                 logRotator {
 378                     artifactDaysToKeep(14)
 379                     daysToKeep(30)
 380                     artifactNumToKeep(100)
 381                     numToKeep(200)
 382                 }
 383                 wrappers {
 384                     timeout {
 385                         absolute(240)
 386                     }
 387                 }
 388             }
 389         } // os
 390
 391         def flowJobPerfRunList = perfOSList.collect { os ->
 392             "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, architecture, isPR)}') }"
 393         }
 394         def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_${architecture}_flow", isPR, '')) {
 395             if (isPR) {
 396                 parameters {
 397                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 398                 }
 399             }
 400             buildFlow("""
 401 // First, build the bits on RHEL7.2
 402 b = build(params, '${fullBuildJobName}')
 403
 404 // Then, run the perf tests
 405 parallel(
 406     ${flowJobPerfRunList.join(",\n    ")}
 407 )
 408 """)
 409         }
 410
 411         Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 412         Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 413
 414         if (isPR) {
 415             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 416             builder.setGithubContext("Linux Perf Test Flow")
 417             builder.triggerOnlyOnComment()
 418             builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 419             builder.triggerForBranch(branch)
 420             builder.emitTrigger(newFlowJob)
 421         }
 422         else {
 423             // Set a push trigger
 424             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 425             builder.emitTrigger(newFlowJob)
 426         }
 427     } // architecture
 428 } // isPR
 429
 430 def static getDockerImageName(def architecture, def os, def isBuild) {
 431     // We must change some docker private images to official later
 432     if (isBuild) {
 433         if (architecture == 'arm') {
 434             if (os == 'Ubuntu') {
 435                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
 436             }
 437         }
 438     }
 439     println("Unknown architecture to use docker: ${architecture} ${os}");
 440     assert false
 441 }
 442
 443 def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
 444     return Utilities.getFullJobName(project, "perf_throughput_${os}_${arch}", isPR)
 445 }
 446
 447 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 448 [true, false].each { isPR ->
 449     ['x64','arm'].each { architecture ->
 450         def fullBuildJobName = Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_build", isPR)
 451         def configuration = 'Release'
 452
 453
 454         def crossCompile = ""
 455         def python = "python3.5"
 456
 457         if (architecture == "arm") {
 458             python = "python3.6"
 459             def buildCommands = []
 460             def newBuildJob = job(fullBuildJobName) {
 461                 def additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
 462                 def dockerImage = getDockerImageName(architecture, 'Ubuntu', true)
 463                 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
 464
 465                 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh release ${architecture} cross crosscomponent"
 466
 467                 steps {
 468                     buildCommands.each { buildCommand ->
 469                         shell(buildCommand)
 470                     }
 471                 }
 472
 473                 publishers {
 474                     azureVMAgentPostBuildAction {
 475                         agentPostBuildAction('Delete agent after build execution (when idle).')
 476                     }
 477                 }
 478             }
 479             Utilities.setMachineAffinity(newBuildJob, "Ubuntu16.04", 'latest-or-auto')
 480             Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 481             Utilities.addArchival(newBuildJob, "bin/Product/**")
 482         }
 483         else {
 484             // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 485             ['RHEL7.2'].each { os ->
 486                 def newBuildJob = job(fullBuildJobName) {
 487                     steps {
 488                         shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
 489                     }
 490                 }
 491                 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 492                 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 493                 Utilities.addArchival(newBuildJob, "bin/Product/**")
 494             }
 495         }
 496
 497         // Actual perf testing on the following OSes
 498         def throughputOSList = ['Ubuntu16.04']
 499         if (architecture == 'arm') {
 500             throughputOSList = ['Ubuntu14.04']
 501         }
 502         def throughputOptLevelList = ['full_opt', 'min_opt']
 503
 504         def throughputOSOptLevelList = []
 505
 506         throughputOSList.each { os ->
 507             throughputOptLevelList.each { opt_level ->
 508                 throughputOSOptLevelList.add("${os}_${opt_level}")
 509             }
 510         }
 511
 512         throughputOSList.each { os ->
 513             throughputOptLevelList.each { opt_level ->
 514                 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", architecture, isPR)) {
 515
 516                     def machineLabel = 'ubuntu_1604_clr_perf'
 517                     if (architecture == 'arm') {
 518                         machineLabel = 'ubuntu_1404_clr_perf_arm'
 519                     }
 520
 521                     label(machineLabel)
 522                         wrappers {
 523                             credentialsBinding {
 524                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 525                             }
 526                         }
 527
 528                     if (isPR) {
 529                         parameters {
 530                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 531                         }
 532                     }
 533
 534                     parameters {
 535                         stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 536                     }
 537
 538                     def osGroup = getOSGroup(os)
 539                     def runType = isPR ? 'private' : 'rolling'
 540                     def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 541                     def archString = architecture == 'arm' ? ' --arch=arm' : ''
 542
 543                     steps {
 544                         shell("bash ./tests/scripts/perf-prep.sh --throughput${archString}")
 545                         copyArtifacts(fullBuildJobName) {
 546                             includePatterns("bin/Product/**")
 547                             buildSelector {
 548                                 buildNumber('\${PRODUCT_BUILD}')
 549                             }
 550                         }
 551                         shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 552                         "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 553                         "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 554                         shell("""${python} ./tests/scripts/run-throughput-perf.py \\
 555                         -arch \"${architecture}\" \\
 556                         -os \"${os}\" \\
 557                         -configuration \"${configuration}\" \\
 558                         -opt_level \"${opt_level}\" \\
 559                         -clr_root \"\${WORKSPACE}\" \\
 560                         -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/lib\" \\
 561                         -run_type \"${runType}\" \\
 562                         -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 563                     }
 564                 }
 565
 566                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 567                 def archiveSettings = new ArchivalSettings()
 568                 archiveSettings.addFiles('throughput-*.csv')
 569                 archiveSettings.addFiles('machinedata.json')
 570                 archiveSettings.setAlwaysArchive()
 571                 Utilities.addArchival(newJob, archiveSettings)
 572
 573                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 574
 575                 // For perf, we need to keep the run results longer
 576                 newJob.with {
 577                     // Enable the log rotator
 578                     logRotator {
 579                         artifactDaysToKeep(7)
 580                         daysToKeep(300)
 581                         artifactNumToKeep(25)
 582                         numToKeep(1000)
 583                     }
 584                 }
 585             } // opt_level
 586         } // os
 587
 588         def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 589             "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, architecture, isPR)}') }"
 590         }
 591         def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_flow", isPR, '')) {
 592             if (isPR) {
 593                 parameters {
 594                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 595                 }
 596             }
 597             buildFlow("""
 598     // First, build the bits on RHEL7.2
 599     b = build(params, '${fullBuildJobName}')
 600
 601     // Then, run the perf tests
 602     parallel(
 603         ${flowJobTPRunList.join(",\n    ")}
 604     )
 605     """)
 606         }
 607
 608         Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 609         Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 610
 611         if (isPR) {
 612             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 613             builder.setGithubContext("Linux ${architecture} Throughput Perf Test Flow")
 614             builder.triggerOnlyOnComment()
 615             builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 616             builder.triggerForBranch(branch)
 617             builder.emitTrigger(newFlowJob)
 618         }
 619         else {
 620             // Set a push trigger
 621             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 622             builder.emitTrigger(newFlowJob)
 623         }
 624     } // architecture
 625 } // isPR
 626
 627 // Setup CoreCLR-Scenarios tests
 628 [true, false].each { isPR ->
 629     ['Windows_NT'].each { os ->
 630         ['x64', 'x86'].each { arch ->
 631             ['ryujit'].each { jit ->
 632                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 633                     def architecture = arch
 634                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 635
 636                         def testEnv = ""
 637
 638                         // Set the label.
 639                         label('windows_server_2016_clr_perf')
 640                         wrappers {
 641                             credentialsBinding {
 642                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 643                             }
 644                         }
 645
 646                         if (isPR) {
 647                             parameters {
 648                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 649                             }
 650                         }
 651
 652                         parameters {
 653                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 654                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 655                         }
 656
 657                         def configuration = 'Release'
 658                         def runType = isPR ? 'private' : 'rolling'
 659                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 660                         def uploadString = '-uploadToBenchview'
 661
 662                         steps {
 663                             // Batch
 664                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 665                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 666                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 667
 668                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 669                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 670                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 671                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 672                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 673                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 674                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 675                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 676                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 677
 678                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 679
 680                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
 681
 682                             // Profile=Off
 683                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 684
 685                             // Profile=On
 686                             if (opt_level != 'min_opt') {
 687                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 688                             }
 689                         }
 690                     }
 691
 692                     def archiveSettings = new ArchivalSettings()
 693                     archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 694                     archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 695                     archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 696                     archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 697                     archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 698                     archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 699                     archiveSettings.addFiles('machinedata.json')
 700                     archiveSettings.setAlwaysArchive()
 701
 702                     Utilities.addArchival(newJob, archiveSettings)
 703                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 704
 705                     newJob.with {
 706                         logRotator {
 707                             artifactDaysToKeep(14)
 708                             daysToKeep(30)
 709                             artifactNumToKeep(100)
 710                             numToKeep(200)
 711                         }
 712                         wrappers {
 713                             timeout {
 714                                 absolute(240)
 715                             }
 716                         }
 717                     }
 718
 719                     if (isPR) {
 720                         def opts = ""
 721                         if (opt_level == 'min_opt') {
 722                             opts = '\\W+min_opts'
 723                         }
 724                         def jitt = ""
 725                         if (jit != 'ryujit') {
 726                             jitt = "\\W+${jit}"
 727                         }
 728
 729                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 730                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 731                         builder.triggerOnlyOnComment()
 732                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 733                         builder.triggerForBranch(branch)
 734                         builder.emitTrigger(newJob)
 735                     }
 736                     else if (opt_level == 'full_opt') {
 737                         // Set a push trigger
 738                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 739                         builder.emitTrigger(newJob)
 740                     }
 741                     else {
 742                         // Set periodic trigger
 743                         Utilities.addPeriodicTrigger(newJob, '@daily')
 744                     }
 745                 }
 746             }
 747         }
 748     }
 749 }
 750
 751 // Setup size-on-disk test
 752 ['Windows_NT'].each { os ->
 753     ['x64', 'x86'].each { arch ->
 754         def architecture = arch
 755         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 756
 757             wrappers {
 758                 credentialsBinding {
 759                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 760                 }
 761             }
 762
 763             def channel = 'master'
 764             def configuration = 'Release'
 765             def runType = 'rolling'
 766             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 767             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 768             def coreRoot = "${testBin}\\Tests\\Core_Root"
 769             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 770
 771             steps {
 772                 // Install nuget and get BenchView tools
 773                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 774                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 775                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 776
 777                 // Generate submission metadata for BenchView
 778                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 779                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 780                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 781                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 782                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 783                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 784                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 785
 786                 // Generate machine data from BenchView
 787                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 788
 789                 // Build CoreCLR and gnerate test layout
 790                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 791                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 792
 793                 // Run the size on disk benchmark
 794                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 795
 796                 // From sodbench.csv, create measurment.json, then submission.json
 797                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 798                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 799
 800                 // If this is a PR, upload submission.json
 801                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 802             }
 803         }
 804
 805         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 806
 807         def archiveSettings = new ArchivalSettings()
 808         archiveSettings.addFiles('bin/toArchive/**')
 809         archiveSettings.addFiles('machinedata.json')
 810         archiveSettings.setAlwaysArchive()
 811
 812         Utilities.addArchival(newJob, archiveSettings)
 813         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 814
 815         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 816         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 817
 818         newJob.with {
 819             logRotator {
 820                 artifactDaysToKeep(14)
 821                 daysToKeep(30)
 822                 artifactNumToKeep(100)
 823                 numToKeep(200)
 824             }
 825             wrappers {
 826                 timeout {
 827                     absolute(240)
 828                 }
 829             }
 830         }
 831     }
 832 }
 833
 834 // Setup IlLink tests
 835 [true, false].each { isPR ->
 836     ['Windows_NT'].each { os ->
 837         ['x64'].each { arch ->
 838             ['ryujit'].each { jit ->
 839                 ['full_opt'].each { opt_level ->
 840                     def architecture = arch
 841                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 842
 843                         def testEnv = ""
 844                         wrappers {
 845                             credentialsBinding {
 846                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 847                             }
 848                         }
 849
 850                         if (isPR) {
 851                             parameters {
 852                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 853                             }
 854                         }
 855
 856                         parameters {
 857                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 858                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 859                         }
 860
 861                         def configuration = 'Release'
 862                         def runType = isPR ? 'private' : 'rolling'
 863                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 864                         def uploadString = '-uploadToBenchview'
 865
 866                         steps {
 867                             // Batch
 868                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 869                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 870                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 871
 872                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 873                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 874                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 875                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 876                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 877                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 878                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 879                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 880                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 881
 882                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 883
 884                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 885
 886                             // Scenario: ILLink
 887                             batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
 888                             "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 889                         }
 890                     }
 891
 892                     def archiveSettings = new ArchivalSettings()
 893                     archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 894                     archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 895                     archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 896                     archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 897                     archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 898                     archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 899                     archiveSettings.addFiles('machinedata.json')
 900                     archiveSettings.setAlwaysArchive()
 901
 902                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 903                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 904                     Utilities.addArchival(newJob, archiveSettings)
 905                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 906
 907                     newJob.with {
 908                         logRotator {
 909                             artifactDaysToKeep(14)
 910                             daysToKeep(30)
 911                             artifactNumToKeep(100)
 912                             numToKeep(200)
 913                         }
 914                         wrappers {
 915                             timeout {
 916                                 absolute(240)
 917                             }
 918                         }
 919                     }
 920
 921                     if (isPR) {
 922                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 923                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 924                         builder.triggerOnlyOnComment()
 925                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 926                         builder.triggerForBranch(branch)
 927                         builder.emitTrigger(newJob)
 928                     }
 929                     else {
 930                         // Set a push trigger
 931                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 932                         builder.emitTrigger(newJob)
 933                     }
 934                 }
 935             }
 936         }
 937     }
 938 }
 939
 940 Utilities.createHelperJob(this, project, branch,
 941     "Welcome to the ${project} Perf help",
 942     "Have a nice day!")