perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 ['ryujit'].each { jit ->
  34                     ['full_opt', 'min_opt'].each { opt_level ->
  35
  36                         def architecture = arch
  37                         def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  38                         def testEnv = ""
  39
  40                         def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  41                             // Set the label.
  42                             if (isSmoketest) {
  43                                 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
  44                             }
  45                             else {
  46                                 label('windows_server_2016_clr_perf')
  47                             }
  48                             wrappers {
  49                                 credentialsBinding {
  50                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  51                                 }
  52                             }
  53
  54                             if (isPR) {
  55                                 parameters {
  56                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  57                                 }
  58                             }
  59
  60                             if (isSmoketest) {
  61                                 parameters {
  62                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  63                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  64                                 }
  65                             }
  66                             else{
  67                                 parameters {
  68                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  69                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  70                                 }
  71                             }
  72
  73                             def configuration = 'Release'
  74                             def runType = isPR ? 'private' : 'rolling'
  75                             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  76                             def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  77
  78                             steps {
  79                                 // Batch
  80
  81                                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  82                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  83                                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  84                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  85                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  86                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  87                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  88                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  89                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  90                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  91                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  92                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  93
  94                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  95
  96                                 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
  97
  98                                 // Run with just stopwatch: Profile=Off
  99                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
 100                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
 101
 102                                 // Run with the full set of counters enabled: Profile=On
 103                                 if (opt_level != 'min_opt') {
 104                                     batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 105                                     batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 106                                 }
 107                             }
 108                         }
 109
 110                         def archiveSettings = new ArchivalSettings()
 111                         archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 112                         archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 113                         archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 114                         archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 115                         archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 116                         archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 117                         archiveSettings.addFiles('machinedata.json')
 118                         archiveSettings.setAlwaysArchive()
 119
 120                         Utilities.addArchival(newJob, archiveSettings)
 121                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 122
 123                         newJob.with {
 124                             logRotator {
 125                                 artifactDaysToKeep(14)
 126                                 daysToKeep(30)
 127                                 artifactNumToKeep(100)
 128                                 numToKeep(200)
 129                             }
 130                             wrappers {
 131                                 timeout {
 132                                     absolute(240)
 133                                 }
 134                             }
 135                         }
 136
 137                         if (isPR) {
 138                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 139                             if (isSmoketest) {
 140                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
 141                             }
 142                             else {
 143                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 144
 145                                 def opts = ""
 146                                 if (opt_level == 'min_opt') {
 147                                     opts = '\\W+min_opts'
 148                                 }
 149                                 def jitt = ""
 150                                 if (jit != 'ryujit') {
 151                                     jitt = "\\W+${jit}"
 152                                 }
 153
 154                                 builder.triggerOnlyOnComment()
 155                                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 156                             }
 157
 158                             builder.triggerForBranch(branch)
 159                             builder.emitTrigger(newJob)
 160                         }
 161                         else if (opt_level == 'full_opt') {
 162                             // Set a push trigger
 163                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 164                             builder.emitTrigger(newJob)
 165                         }
 166                         else {
 167                             // Set periodic trigger
 168                             Utilities.addPeriodicTrigger(newJob, '@daily')
 169                         }
 170                     }
 171                 }
 172             }
 173         }
 174     }
 175 }
 176
 177 // Setup throughput perflab tests runs
 178 [true, false].each { isPR ->
 179     ['Windows_NT'].each { os ->
 180         ['x64', 'x86'].each { arch ->
 181             ['ryujit'].each { jit ->
 182                 [true, false].each { pgo_optimized ->
 183                     ['full_opt', 'min_opt'].each { opt_level ->
 184                         def architecture = arch
 185
 186                         pgo_build = ""
 187                         pgo_test = ""
 188                         pgo_string = "pgo"
 189                         if (!pgo_optimized) {
 190                             pgo_build = " -nopgooptimize"
 191                             pgo_test = " -nopgo"
 192                             pgo_string = "nopgo"
 193                         }
 194
 195                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 196                             // Set the label.
 197                             label('windows_server_2016_clr_perf')
 198                             wrappers {
 199                                 credentialsBinding {
 200                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 201                                 }
 202                             }
 203
 204                             if (isPR) {
 205                                 parameters {
 206                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 207                                 }
 208                             }
 209
 210                             def configuration = 'Release'
 211                             def runType = isPR ? 'private' : 'rolling'
 212                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 213
 214                             steps {
 215                                 // Batch
 216                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 217                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 218                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 219                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 220                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 221                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 222                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 223                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 224                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 225                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 226                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 227                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 228                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 229                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 230                             }
 231                         }
 232
 233                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 234                         def archiveSettings = new ArchivalSettings()
 235                         archiveSettings.addFiles('throughput-*.csv')
 236                         archiveSettings.setAlwaysArchive()
 237                         Utilities.addArchival(newJob, archiveSettings)
 238
 239                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 240
 241                         if (isPR) {
 242                             def opts = ""
 243                             if (opt_level == 'min_opt') {
 244                                 opts = '\\W+min_opts'
 245                             }
 246
 247                             def jitt = ""
 248                             if (jit != 'ryujit') {
 249                                 jitt = "\\W+${jit}"
 250                             }
 251
 252                             def pgo_trigger = ""
 253                             if (pgo_optimized) {
 254                                 pgo_trigger = "\\W+nopgo"
 255                             }
 256
 257
 258                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 259                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 260                             builder.triggerOnlyOnComment()
 261                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 262                             builder.triggerForBranch(branch)
 263                             builder.emitTrigger(newJob)
 264                         }
 265                         else if (opt_level == 'full_opt' && pgo_optimized) {
 266                             // Set a push trigger
 267                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 268                             builder.emitTrigger(newJob)
 269                         }
 270                         else {
 271                             // Set periodic trigger
 272                             Utilities.addPeriodicTrigger(newJob, '@daily')
 273                         }
 274                     }
 275                 }
 276             }
 277         }
 278     }
 279 }
 280
 281 def static getFullPerfJobName(def project, def os, def arch, def isPR) {
 282     return Utilities.getFullJobName(project, "perf_${os}_${arch}", isPR)
 283 }
 284
 285 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 286 [true, false].each { isPR ->
 287     ['x64'].each { architecture ->
 288         def fullBuildJobName = Utilities.getFullJobName(project, "perf_linux_build", isPR)
 289         def configuration = 'Release'
 290
 291         def crossCompile = ""
 292         def crossLayout = ""
 293         def python = "python3.5"
 294
 295         // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 296         ['RHEL7.2'].each { os ->
 297             def newBuildJob = job(fullBuildJobName) {
 298                 steps {
 299                     shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
 300                     shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}${crossLayout}")
 301                 }
 302             }
 303             Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 304             Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 305             Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
 306         }
 307
 308
 309         // Actual perf testing on the following OSes
 310         def perfOSList = ['Ubuntu16.04']
 311
 312         perfOSList.each { os ->
 313             def newJob = job(getFullPerfJobName(project, os, architecture, isPR)) {
 314
 315                 def machineLabel = 'ubuntu_1604_clr_perf'
 316
 317                 label(machineLabel)
 318                 wrappers {
 319                     credentialsBinding {
 320                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 321                     }
 322                 }
 323
 324                 if (isPR) {
 325                     parameters {
 326                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 327                     }
 328                 }
 329
 330                 parameters {
 331                     // Cap the maximum number of iterations to 21.
 332                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 333                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 334                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 335                 }
 336
 337                 def osGroup = getOSGroup(os)
 338                 def runType = isPR ? 'private' : 'rolling'
 339                 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 340                 def uploadString = '-uploadToBenchview'
 341
 342                 def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
 343
 344                 steps {
 345                     shell("./tests/scripts/perf-prep.sh --nocorefx")
 346                     shell("./init-tools.sh")
 347                     copyArtifacts(fullBuildJobName) {
 348                         includePatterns("bin/**")
 349                         buildSelector {
 350                             buildNumber('\${PRODUCT_BUILD}')
 351                         }
 352                     }
 353                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 354                     "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 355                     "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 356                     shell("""${python} ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
 357                 }
 358             }
 359
 360             def archiveSettings = new ArchivalSettings()
 361             archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 362             archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 363             archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 364             archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 365             archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 366             archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 367             archiveSettings.addFiles('machinedata.json')
 368             archiveSettings.setAlwaysArchive()
 369
 370             Utilities.addArchival(newJob, archiveSettings)
 371             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 372
 373             // For perf, we need to keep the run results longer
 374             newJob.with {
 375                 // Enable the log rotator
 376                 logRotator {
 377                     artifactDaysToKeep(14)
 378                     daysToKeep(30)
 379                     artifactNumToKeep(100)
 380                     numToKeep(200)
 381                 }
 382                 wrappers {
 383                     timeout {
 384                         absolute(240)
 385                     }
 386                 }
 387             }
 388         } // os
 389
 390         def flowJobPerfRunList = perfOSList.collect { os ->
 391             "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, architecture, isPR)}') }"
 392         }
 393         def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_${architecture}_flow", isPR, '')) {
 394             if (isPR) {
 395                 parameters {
 396                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 397                 }
 398             }
 399             buildFlow("""
 400 // First, build the bits on RHEL7.2
 401 b = build(params, '${fullBuildJobName}')
 402
 403 // Then, run the perf tests
 404 parallel(
 405     ${flowJobPerfRunList.join(",\n    ")}
 406 )
 407 """)
 408         }
 409
 410         Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 411         Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 412
 413         if (isPR) {
 414             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 415             builder.setGithubContext("Linux Perf Test Flow")
 416             builder.triggerOnlyOnComment()
 417             builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 418             builder.triggerForBranch(branch)
 419             builder.emitTrigger(newFlowJob)
 420         }
 421         else {
 422             // Set a push trigger
 423             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 424             builder.emitTrigger(newFlowJob)
 425         }
 426     } // architecture
 427 } // isPR
 428
 429 def static getDockerImageName(def architecture, def os, def isBuild) {
 430     // We must change some docker private images to official later
 431     if (isBuild) {
 432         if (architecture == 'arm') {
 433             if (os == 'Ubuntu') {
 434                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
 435             }
 436         }
 437     }
 438     println("Unknown architecture to use docker: ${architecture} ${os}");
 439     assert false
 440 }
 441
 442 def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
 443     return Utilities.getFullJobName(project, "perf_throughput_${os}_${arch}", isPR)
 444 }
 445
 446 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 447 [true, false].each { isPR ->
 448     ['x64','arm'].each { architecture ->
 449         def fullBuildJobName = Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_build", isPR)
 450         def configuration = 'Release'
 451
 452
 453         def crossCompile = ""
 454         def python = "python3.5"
 455
 456         if (architecture == "arm") {
 457             python = "python3.6"
 458             def buildCommands = []
 459             def newBuildJob = job(fullBuildJobName) {
 460                 def additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
 461                 def dockerImage = getDockerImageName(architecture, 'Ubuntu', true)
 462                 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
 463
 464                 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh release ${architecture} cross crosscomponent"
 465
 466                 steps {
 467                     buildCommands.each { buildCommand ->
 468                         shell(buildCommand)
 469                     }
 470                 }
 471
 472                 publishers {
 473                     azureVMAgentPostBuildAction {
 474                         agentPostBuildAction('Delete agent after build execution (when idle).')
 475                     }
 476                 }
 477             }
 478             Utilities.setMachineAffinity(newBuildJob, "Ubuntu16.04", 'latest-or-auto')
 479             Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 480             Utilities.addArchival(newBuildJob, "bin/Product/**")
 481         }
 482         else {
 483             // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 484             ['RHEL7.2'].each { os ->
 485                 def newBuildJob = job(fullBuildJobName) {
 486                     steps {
 487                         shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
 488                     }
 489                 }
 490                 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 491                 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 492                 Utilities.addArchival(newBuildJob, "bin/Product/**")
 493             }
 494         }
 495
 496         // Actual perf testing on the following OSes
 497         def throughputOSList = ['Ubuntu16.04']
 498         if (architecture == 'arm') {
 499             throughputOSList = ['Ubuntu14.04']
 500         }
 501         def throughputOptLevelList = ['full_opt', 'min_opt']
 502
 503         def throughputOSOptLevelList = []
 504
 505         throughputOSList.each { os ->
 506             throughputOptLevelList.each { opt_level ->
 507                 throughputOSOptLevelList.add("${os}_${opt_level}")
 508             }
 509         }
 510
 511         throughputOSList.each { os ->
 512             throughputOptLevelList.each { opt_level ->
 513                 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", architecture, isPR)) {
 514
 515                     def machineLabel = 'ubuntu_1604_clr_perf'
 516                     if (architecture == 'arm') {
 517                         machineLabel = 'ubuntu_1404_clr_perf_arm'
 518                     }
 519
 520                     label(machineLabel)
 521                         wrappers {
 522                             credentialsBinding {
 523                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 524                             }
 525                         }
 526
 527                     if (isPR) {
 528                         parameters {
 529                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 530                         }
 531                     }
 532
 533                     parameters {
 534                         stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 535                     }
 536
 537                     def osGroup = getOSGroup(os)
 538                     def runType = isPR ? 'private' : 'rolling'
 539                     def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 540                     def archString = architecture == 'arm' ? ' --arch=arm' : ''
 541
 542                     steps {
 543                         shell("bash ./tests/scripts/perf-prep.sh --throughput${archString}")
 544                         copyArtifacts(fullBuildJobName) {
 545                             includePatterns("bin/Product/**")
 546                             buildSelector {
 547                                 buildNumber('\${PRODUCT_BUILD}')
 548                             }
 549                         }
 550                         shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 551                         "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 552                         "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 553                         shell("""${python} ./tests/scripts/run-throughput-perf.py \\
 554                         -arch \"${architecture}\" \\
 555                         -os \"${os}\" \\
 556                         -configuration \"${configuration}\" \\
 557                         -opt_level \"${opt_level}\" \\
 558                         -clr_root \"\${WORKSPACE}\" \\
 559                         -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/lib\" \\
 560                         -run_type \"${runType}\" \\
 561                         -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 562                     }
 563                 }
 564
 565                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 566                 def archiveSettings = new ArchivalSettings()
 567                 archiveSettings.addFiles('throughput-*.csv')
 568                 archiveSettings.addFiles('machinedata.json')
 569                 archiveSettings.setAlwaysArchive()
 570                 Utilities.addArchival(newJob, archiveSettings)
 571
 572                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 573
 574                 // For perf, we need to keep the run results longer
 575                 newJob.with {
 576                     // Enable the log rotator
 577                     logRotator {
 578                         artifactDaysToKeep(7)
 579                         daysToKeep(300)
 580                         artifactNumToKeep(25)
 581                         numToKeep(1000)
 582                     }
 583                 }
 584             } // opt_level
 585         } // os
 586
 587         def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 588             "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, architecture, isPR)}') }"
 589         }
 590         def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_flow", isPR, '')) {
 591             if (isPR) {
 592                 parameters {
 593                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 594                 }
 595             }
 596             buildFlow("""
 597     // First, build the bits on RHEL7.2
 598     b = build(params, '${fullBuildJobName}')
 599
 600     // Then, run the perf tests
 601     parallel(
 602         ${flowJobTPRunList.join(",\n    ")}
 603     )
 604     """)
 605         }
 606
 607         Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 608         Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 609
 610         if (isPR) {
 611             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 612             builder.setGithubContext("Linux ${architecture} Throughput Perf Test Flow")
 613             builder.triggerOnlyOnComment()
 614             builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 615             builder.triggerForBranch(branch)
 616             builder.emitTrigger(newFlowJob)
 617         }
 618         else {
 619             // Set a push trigger
 620             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 621             builder.emitTrigger(newFlowJob)
 622         }
 623     } // architecture
 624 } // isPR
 625
 626 // Setup CoreCLR-Scenarios tests
 627 [true, false].each { isPR ->
 628     ['Windows_NT'].each { os ->
 629         ['x64', 'x86'].each { arch ->
 630             ['ryujit'].each { jit ->
 631                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 632                     def architecture = arch
 633                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 634
 635                         def testEnv = ""
 636
 637                         // Set the label.
 638                         label('windows_server_2016_clr_perf')
 639                         wrappers {
 640                             credentialsBinding {
 641                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 642                             }
 643                         }
 644
 645                         if (isPR) {
 646                             parameters {
 647                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 648                             }
 649                         }
 650
 651                         parameters {
 652                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 653                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 654                         }
 655
 656                         def configuration = 'Release'
 657                         def runType = isPR ? 'private' : 'rolling'
 658                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 659                         def uploadString = '-uploadToBenchview'
 660
 661                         steps {
 662                             // Batch
 663                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 664                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 665                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 666
 667                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 668                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 669                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 670                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 671                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 672                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 673                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 674                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 675                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 676
 677                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 678
 679                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
 680
 681                             // Profile=Off
 682                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 683
 684                             // Profile=On
 685                             if (opt_level != 'min_opt') {
 686                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 687                             }
 688                         }
 689                     }
 690
 691                     def archiveSettings = new ArchivalSettings()
 692                     archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 693                     archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 694                     archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 695                     archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 696                     archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 697                     archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 698                     archiveSettings.addFiles('machinedata.json')
 699                     archiveSettings.setAlwaysArchive()
 700
 701                     Utilities.addArchival(newJob, archiveSettings)
 702                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 703
 704                     newJob.with {
 705                         logRotator {
 706                             artifactDaysToKeep(14)
 707                             daysToKeep(30)
 708                             artifactNumToKeep(100)
 709                             numToKeep(200)
 710                         }
 711                         wrappers {
 712                             timeout {
 713                                 absolute(240)
 714                             }
 715                         }
 716                     }
 717
 718                     if (isPR) {
 719                         def opts = ""
 720                         if (opt_level == 'min_opt') {
 721                             opts = '\\W+min_opts'
 722                         }
 723                         def jitt = ""
 724                         if (jit != 'ryujit') {
 725                             jitt = "\\W+${jit}"
 726                         }
 727
 728                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 729                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 730                         builder.triggerOnlyOnComment()
 731                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 732                         builder.triggerForBranch(branch)
 733                         builder.emitTrigger(newJob)
 734                     }
 735                     else if (opt_level == 'full_opt') {
 736                         // Set a push trigger
 737                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 738                         builder.emitTrigger(newJob)
 739                     }
 740                     else {
 741                         // Set periodic trigger
 742                         Utilities.addPeriodicTrigger(newJob, '@daily')
 743                     }
 744                 }
 745             }
 746         }
 747     }
 748 }
 749
 750 // Setup size-on-disk test
 751 ['Windows_NT'].each { os ->
 752     ['x64', 'x86'].each { arch ->
 753         def architecture = arch
 754         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 755
 756             wrappers {
 757                 credentialsBinding {
 758                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 759                 }
 760             }
 761
 762             def channel = 'master'
 763             def configuration = 'Release'
 764             def runType = 'rolling'
 765             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 766             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 767             def coreRoot = "${testBin}\\Tests\\Core_Root"
 768             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 769
 770             steps {
 771                 // Install nuget and get BenchView tools
 772                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 773                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 774                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 775
 776                 // Generate submission metadata for BenchView
 777                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 778                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 779                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 780                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 781                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 782                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 783                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 784
 785                 // Generate machine data from BenchView
 786                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 787
 788                 // Build CoreCLR and gnerate test layout
 789                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 790                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 791
 792                 // Run the size on disk benchmark
 793                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 794
 795                 // From sodbench.csv, create measurment.json, then submission.json
 796                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 797                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 798
 799                 // If this is a PR, upload submission.json
 800                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 801             }
 802         }
 803
 804         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 805
 806         def archiveSettings = new ArchivalSettings()
 807         archiveSettings.addFiles('bin/toArchive/**')
 808         archiveSettings.addFiles('machinedata.json')
 809         archiveSettings.setAlwaysArchive()
 810
 811         Utilities.addArchival(newJob, archiveSettings)
 812         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 813
 814         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 815         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 816
 817         newJob.with {
 818             logRotator {
 819                 artifactDaysToKeep(14)
 820                 daysToKeep(30)
 821                 artifactNumToKeep(100)
 822                 numToKeep(200)
 823             }
 824             wrappers {
 825                 timeout {
 826                     absolute(240)
 827                 }
 828             }
 829         }
 830     }
 831 }
 832
 833 // Setup IlLink tests
 834 [true, false].each { isPR ->
 835     ['Windows_NT'].each { os ->
 836         ['x64'].each { arch ->
 837             ['ryujit'].each { jit ->
 838                 ['full_opt'].each { opt_level ->
 839                     def architecture = arch
 840                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 841
 842                         def testEnv = ""
 843                         wrappers {
 844                             credentialsBinding {
 845                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 846                             }
 847                         }
 848
 849                         if (isPR) {
 850                             parameters {
 851                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 852                             }
 853                         }
 854
 855                         parameters {
 856                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 857                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 858                         }
 859
 860                         def configuration = 'Release'
 861                         def runType = isPR ? 'private' : 'rolling'
 862                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 863                         def uploadString = '-uploadToBenchview'
 864
 865                         steps {
 866                             // Batch
 867                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 868                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 869                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 870
 871                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 872                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 873                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 874                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 875                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 876                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 877                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 878                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 879                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 880
 881                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 882
 883                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 884
 885                             // Scenario: ILLink
 886                             batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
 887                             "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 888                         }
 889                     }
 890
 891                     def archiveSettings = new ArchivalSettings()
 892                     archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 893                     archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 894                     archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 895                     archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 896                     archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 897                     archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 898                     archiveSettings.addFiles('machinedata.json')
 899                     archiveSettings.setAlwaysArchive()
 900
 901                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 902                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 903                     Utilities.addArchival(newJob, archiveSettings)
 904                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 905
 906                     newJob.with {
 907                         logRotator {
 908                             artifactDaysToKeep(14)
 909                             daysToKeep(30)
 910                             artifactNumToKeep(100)
 911                             numToKeep(200)
 912                         }
 913                         wrappers {
 914                             timeout {
 915                                 absolute(240)
 916                             }
 917                         }
 918                     }
 919
 920                     if (isPR) {
 921                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 922                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 923                         builder.triggerOnlyOnComment()
 924                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 925                         builder.triggerForBranch(branch)
 926                         builder.emitTrigger(newJob)
 927                     }
 928                     else {
 929                         // Set a push trigger
 930                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 931                         builder.emitTrigger(newJob)
 932                     }
 933                 }
 934             }
 935         }
 936     }
 937 }
 938
 939 Utilities.createHelperJob(this, project, branch,
 940     "Welcome to the ${project} Perf help",
 941     "Have a nice day!")