perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 ['ryujit'].each { jit ->
  34                     ['full_opt', 'min_opt'].each { opt_level ->
  35
  36                         def architecture = arch
  37                         def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  38                         def testEnv = ""
  39                         def python = "C:\\Python35\\python.exe"
  40
  41                         def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  42                             // Set the label.
  43                             if (isSmoketest) {
  44                                 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
  45                                 python = "C:\\python3.7.0\\python.exe"
  46                             }
  47                             else {
  48                                 label('windows_server_2016_clr_perf')
  49                             }
  50
  51                             wrappers {
  52                                 credentialsBinding {
  53                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  54                                 }
  55                             }
  56
  57                             if (isPR) {
  58                                 parameters {
  59                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  60                                 }
  61                             }
  62
  63                             if (isSmoketest) {
  64                                 parameters {
  65                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  66                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  67                                 }
  68                             }
  69                             else{
  70                                 parameters {
  71                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  72                                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  73                                 }
  74                             }
  75
  76                             def configuration = 'Release'
  77                             def runType = isPR ? 'private' : 'rolling'
  78                             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  79                             def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  80
  81                             steps {
  82                                 // Batch
  83
  84                                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  85                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  86                                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  87                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  88                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  89                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  90                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  91                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  92                                 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  93                                 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  94                                 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  95                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  96
  97                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  98
  99                                 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
 100
 101                                 // Run with just stopwatch: Profile=Off
 102                                 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
 103                                 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
 104
 105                                 // Run with the full set of counters enabled: Profile=On
 106                                 if (opt_level != 'min_opt') {
 107                                     batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 108                                     batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 109                                 }
 110                             }
 111                         }
 112
 113                         def archiveSettings = new ArchivalSettings()
 114                         archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 115                         archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 116                         archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 117                         archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 118                         archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 119                         archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 120                         archiveSettings.addFiles('machinedata.json')
 121                         archiveSettings.setAlwaysArchive()
 122
 123                         Utilities.addArchival(newJob, archiveSettings)
 124                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 125
 126                         newJob.with {
 127                             logRotator {
 128                                 artifactDaysToKeep(14)
 129                                 daysToKeep(30)
 130                                 artifactNumToKeep(100)
 131                                 numToKeep(200)
 132                             }
 133                             wrappers {
 134                                 timeout {
 135                                     absolute(240)
 136                                 }
 137                             }
 138                         }
 139
 140                         if (isPR) {
 141                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 142                             if (isSmoketest) {
 143                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
 144                             }
 145                             else {
 146                                 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 147
 148                                 def opts = ""
 149                                 if (opt_level == 'min_opt') {
 150                                     opts = '\\W+min_opts'
 151                                 }
 152                                 def jitt = ""
 153                                 if (jit != 'ryujit') {
 154                                     jitt = "\\W+${jit}"
 155                                 }
 156
 157                                 builder.triggerOnlyOnComment()
 158                                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 159                             }
 160
 161                             builder.triggerForBranch(branch)
 162                             builder.emitTrigger(newJob)
 163                         }
 164                         else if (opt_level == 'full_opt') {
 165                             // Set a push trigger
 166                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 167                             builder.emitTrigger(newJob)
 168                         }
 169                         else {
 170                             // Set periodic trigger
 171                             Utilities.addPeriodicTrigger(newJob, '@daily')
 172                         }
 173                     }
 174                 }
 175             }
 176         }
 177     }
 178 }
 179
 180 // Setup throughput perflab tests runs
 181 [true, false].each { isPR ->
 182     ['Windows_NT'].each { os ->
 183         ['x64', 'x86'].each { arch ->
 184             ['ryujit'].each { jit ->
 185                 [true, false].each { pgo_optimized ->
 186                     ['full_opt', 'min_opt'].each { opt_level ->
 187                         def architecture = arch
 188
 189                         def python = "C:\\Python35\\python.exe"
 190
 191                         pgo_build = ""
 192                         pgo_test = ""
 193                         pgo_string = "pgo"
 194                         if (!pgo_optimized) {
 195                             pgo_build = " -nopgooptimize"
 196                             pgo_test = " -nopgo"
 197                             pgo_string = "nopgo"
 198                         }
 199
 200                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 201                             // Set the label.
 202                             label('windows_server_2016_clr_perf')
 203                             wrappers {
 204                                 credentialsBinding {
 205                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 206                                 }
 207                             }
 208
 209                             if (isPR) {
 210                                 parameters {
 211                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 212                                 }
 213                             }
 214
 215                             def configuration = 'Release'
 216                             def runType = isPR ? 'private' : 'rolling'
 217                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 218
 219                             steps {
 220                                 // Batch
 221                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 222                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 223                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 224                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 225                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 226                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 227                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 228                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 229                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 230                                 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 231                                 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 232                                 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 233                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 234                                 batchFile("${python} -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 235                             }
 236                         }
 237
 238                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 239                         def archiveSettings = new ArchivalSettings()
 240                         archiveSettings.addFiles('throughput-*.csv')
 241                         archiveSettings.setAlwaysArchive()
 242                         Utilities.addArchival(newJob, archiveSettings)
 243
 244                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 245
 246                         if (isPR) {
 247                             def opts = ""
 248                             if (opt_level == 'min_opt') {
 249                                 opts = '\\W+min_opts'
 250                             }
 251
 252                             def jitt = ""
 253                             if (jit != 'ryujit') {
 254                                 jitt = "\\W+${jit}"
 255                             }
 256
 257                             def pgo_trigger = ""
 258                             if (pgo_optimized) {
 259                                 pgo_trigger = "\\W+nopgo"
 260                             }
 261
 262
 263                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 264                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 265                             builder.triggerOnlyOnComment()
 266                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 267                             builder.triggerForBranch(branch)
 268                             builder.emitTrigger(newJob)
 269                         }
 270                         else if (opt_level == 'full_opt' && pgo_optimized) {
 271                             // Set a push trigger
 272                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 273                             builder.emitTrigger(newJob)
 274                         }
 275                         else {
 276                             // Set periodic trigger
 277                             Utilities.addPeriodicTrigger(newJob, '@daily')
 278                         }
 279                     }
 280                 }
 281             }
 282         }
 283     }
 284 }
 285
 286 def static getFullPerfJobName(def project, def os, def arch, def isPR) {
 287     return Utilities.getFullJobName(project, "perf_${os}_${arch}", isPR)
 288 }
 289
 290 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 291 [true, false].each { isPR ->
 292     ['x64'].each { architecture ->
 293         def fullBuildJobName = Utilities.getFullJobName(project, "perf_linux_build", isPR)
 294         def configuration = 'Release'
 295
 296         def crossCompile = ""
 297         def crossLayout = ""
 298         def python = "python3.5"
 299
 300         // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 301         ['RHEL7.2'].each { os ->
 302             def newBuildJob = job(fullBuildJobName) {
 303                 steps {
 304                     shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
 305                     shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}${crossLayout}")
 306                 }
 307             }
 308             Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 309             Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 310             Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
 311         }
 312
 313
 314         // Actual perf testing on the following OSes
 315         def perfOSList = ['Ubuntu16.04']
 316
 317         perfOSList.each { os ->
 318             def newJob = job(getFullPerfJobName(project, os, architecture, isPR)) {
 319
 320                 def machineLabel = 'ubuntu_1604_clr_perf'
 321
 322                 label(machineLabel)
 323                 wrappers {
 324                     credentialsBinding {
 325                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 326                     }
 327                 }
 328
 329                 if (isPR) {
 330                     parameters {
 331                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 332                     }
 333                 }
 334
 335                 parameters {
 336                     // Cap the maximum number of iterations to 21.
 337                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 338                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 339                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 340                 }
 341
 342                 def osGroup = getOSGroup(os)
 343                 def runType = isPR ? 'private' : 'rolling'
 344                 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 345                 def uploadString = '-uploadToBenchview'
 346
 347                 def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
 348
 349                 steps {
 350                     shell("./tests/scripts/perf-prep.sh --nocorefx")
 351                     shell("./init-tools.sh")
 352                     copyArtifacts(fullBuildJobName) {
 353                         includePatterns("bin/**")
 354                         buildSelector {
 355                             buildNumber('\${PRODUCT_BUILD}')
 356                         }
 357                     }
 358                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 359                     "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 360                     "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 361                     shell("""${python} ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
 362                 }
 363             }
 364
 365             def archiveSettings = new ArchivalSettings()
 366             archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 367             archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 368             archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 369             archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 370             archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 371             archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 372             archiveSettings.addFiles('machinedata.json')
 373             archiveSettings.setAlwaysArchive()
 374
 375             Utilities.addArchival(newJob, archiveSettings)
 376             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 377
 378             // For perf, we need to keep the run results longer
 379             newJob.with {
 380                 // Enable the log rotator
 381                 logRotator {
 382                     artifactDaysToKeep(14)
 383                     daysToKeep(30)
 384                     artifactNumToKeep(100)
 385                     numToKeep(200)
 386                 }
 387                 wrappers {
 388                     timeout {
 389                         absolute(240)
 390                     }
 391                 }
 392             }
 393         } // os
 394
 395         def flowJobPerfRunList = perfOSList.collect { os ->
 396             "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, architecture, isPR)}') }"
 397         }
 398         def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_${architecture}_flow", isPR, '')) {
 399             if (isPR) {
 400                 parameters {
 401                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 402                 }
 403             }
 404             buildFlow("""
 405 // First, build the bits on RHEL7.2
 406 b = build(params, '${fullBuildJobName}')
 407
 408 // Then, run the perf tests
 409 parallel(
 410     ${flowJobPerfRunList.join(",\n    ")}
 411 )
 412 """)
 413         }
 414
 415         Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 416         Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 417
 418         if (isPR) {
 419             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 420             builder.setGithubContext("Linux Perf Test Flow")
 421             builder.triggerOnlyOnComment()
 422             builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 423             builder.triggerForBranch(branch)
 424             builder.emitTrigger(newFlowJob)
 425         }
 426         else {
 427             // Set a push trigger
 428             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 429             builder.emitTrigger(newFlowJob)
 430         }
 431     } // architecture
 432 } // isPR
 433
 434 def static getDockerImageName(def architecture, def os, def isBuild) {
 435     // We must change some docker private images to official later
 436     if (isBuild) {
 437         if (architecture == 'arm') {
 438             if (os == 'Ubuntu') {
 439                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
 440             }
 441         }
 442     }
 443     println("Unknown architecture to use docker: ${architecture} ${os}");
 444     assert false
 445 }
 446
 447 def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
 448     return Utilities.getFullJobName(project, "perf_throughput_${os}_${arch}", isPR)
 449 }
 450
 451 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 452 [true, false].each { isPR ->
 453     ['x64','arm'].each { architecture ->
 454         def fullBuildJobName = Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_build", isPR)
 455         def configuration = 'Release'
 456
 457
 458         def crossCompile = ""
 459         def python = "python3.5"
 460         def tgzFile = "bin-Product-Linux.${architecture}.${configuration}.tgz"
 461
 462         if (architecture == "arm") {
 463             python = "python3.6"
 464             def buildCommands = []
 465             def newBuildJob = job(fullBuildJobName) {
 466                 def additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
 467                 def dockerImage = getDockerImageName(architecture, 'Ubuntu', true)
 468                 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
 469
 470                 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh release ${architecture} cross"
 471
 472                 steps {
 473                     buildCommands.each { buildCommand ->
 474                         shell(buildCommand)
 475                         shell("tar -czf ${tgzFile} bin/Product/Linux.${architecture}.${configuration}")
 476                     }
 477                 }
 478
 479                 publishers {
 480                     azureVMAgentPostBuildAction {
 481                         agentPostBuildAction('Delete agent after build execution (when idle).')
 482                     }
 483                 }
 484             }
 485             Utilities.setMachineAffinity(newBuildJob, "Ubuntu16.04", 'latest-or-auto')
 486             Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 487             Utilities.addArchival(newBuildJob, "${tgzFile}")
 488         }
 489         else {
 490             // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 491             ['RHEL7.2'].each { os ->
 492                 def newBuildJob = job(fullBuildJobName) {
 493                     steps {
 494                         shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
 495                     }
 496                 }
 497                 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 498                 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 499                 Utilities.addArchival(newBuildJob, "bin/Product/**")
 500             }
 501         }
 502
 503         // Actual perf testing on the following OSes
 504         def throughputOSList = ['Ubuntu16.04']
 505         if (architecture == 'arm') {
 506             throughputOSList = ['Ubuntu14.04']
 507         }
 508         def throughputOptLevelList = ['full_opt', 'min_opt']
 509
 510         def throughputOSOptLevelList = []
 511
 512         throughputOSList.each { os ->
 513             throughputOptLevelList.each { opt_level ->
 514                 throughputOSOptLevelList.add("${os}_${opt_level}")
 515             }
 516         }
 517
 518         throughputOSList.each { os ->
 519             throughputOptLevelList.each { opt_level ->
 520                 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", architecture, isPR)) {
 521
 522                     def machineLabel = 'ubuntu_1604_clr_perf'
 523                     if (architecture == 'arm') {
 524                         machineLabel = 'ubuntu_1404_clr_perf_arm'
 525                     }
 526
 527                     label(machineLabel)
 528                         wrappers {
 529                             credentialsBinding {
 530                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 531                             }
 532                         }
 533
 534                     if (isPR) {
 535                         parameters {
 536                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 537                         }
 538                     }
 539
 540                     parameters {
 541                         stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 542                     }
 543
 544                     def osGroup = getOSGroup(os)
 545                     def runType = isPR ? 'private' : 'rolling'
 546                     def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 547                     def archString = architecture == 'arm' ? ' --arch=arm' : ''
 548
 549                     steps {
 550                         shell("bash ./tests/scripts/perf-prep.sh --throughput${archString}")
 551                         copyArtifacts(fullBuildJobName) {
 552                             if (architecture == 'arm') {
 553                                 includePatterns("${tgzFile}")
 554                             }
 555                             else {
 556                                 includePatterns("bin/Product/**")
 557                             }
 558                             buildSelector {
 559                                 buildNumber('\${PRODUCT_BUILD}')
 560                             }
 561                         }
 562                         if (architecture == 'arm') {
 563                             shell("tar -xzf ./${tgzFile} || exit 0")
 564                         }
 565                         shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 566                         "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 567                         "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 568                         shell("""${python} ./tests/scripts/run-throughput-perf.py \\
 569                         -arch \"${architecture}\" \\
 570                         -os \"${os}\" \\
 571                         -configuration \"${configuration}\" \\
 572                         -opt_level \"${opt_level}\" \\
 573                         -clr_root \"\${WORKSPACE}\" \\
 574                         -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/lib\" \\
 575                         -run_type \"${runType}\" \\
 576                         -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 577                     }
 578                 }
 579
 580                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 581                 def archiveSettings = new ArchivalSettings()
 582                 archiveSettings.addFiles('throughput-*.csv')
 583                 archiveSettings.addFiles('machinedata.json')
 584                 archiveSettings.setAlwaysArchive()
 585                 Utilities.addArchival(newJob, archiveSettings)
 586
 587                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 588
 589                 // For perf, we need to keep the run results longer
 590                 newJob.with {
 591                     // Enable the log rotator
 592                     logRotator {
 593                         artifactDaysToKeep(7)
 594                         daysToKeep(300)
 595                         artifactNumToKeep(25)
 596                         numToKeep(1000)
 597                     }
 598                 }
 599             } // opt_level
 600         } // os
 601
 602         def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 603             "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, architecture, isPR)}') }"
 604         }
 605         def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_flow", isPR, '')) {
 606             if (isPR) {
 607                 parameters {
 608                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 609                 }
 610             }
 611             buildFlow("""
 612     // First, build the bits on RHEL7.2
 613     b = build(params, '${fullBuildJobName}')
 614
 615     // Then, run the perf tests
 616     parallel(
 617         ${flowJobTPRunList.join(",\n    ")}
 618     )
 619     """)
 620         }
 621
 622         Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 623         Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 624
 625         if (isPR) {
 626             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 627             builder.setGithubContext("Linux ${architecture} Throughput Perf Test Flow")
 628             builder.triggerOnlyOnComment()
 629             builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 630             builder.triggerForBranch(branch)
 631             builder.emitTrigger(newFlowJob)
 632         }
 633         else {
 634             // Set a push trigger
 635             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 636             builder.emitTrigger(newFlowJob)
 637         }
 638     } // architecture
 639 } // isPR
 640
 641 // Setup CoreCLR-Scenarios tests
 642 [true, false].each { isPR ->
 643     ['Windows_NT'].each { os ->
 644         ['x64', 'x86'].each { arch ->
 645             ['ryujit'].each { jit ->
 646                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 647                     def architecture = arch
 648                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 649
 650                         def testEnv = ""
 651                         def python = "C:\\Python35\\python.exe"
 652
 653                         // Set the label.
 654                         label('windows_server_2016_clr_perf')
 655                         wrappers {
 656                             credentialsBinding {
 657                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 658                             }
 659                         }
 660
 661                         if (isPR) {
 662                             parameters {
 663                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 664                             }
 665                         }
 666
 667                         parameters {
 668                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 669                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 670                         }
 671
 672                         def configuration = 'Release'
 673                         def runType = isPR ? 'private' : 'rolling'
 674                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 675                         def uploadString = '-uploadToBenchview'
 676
 677                         steps {
 678                             // Batch
 679                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 680                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 681                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 682
 683                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 684                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 685                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 686                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 687                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 688                             "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 689                             "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 690                             batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 691                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 692
 693                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 694
 695                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
 696
 697                             // Profile=Off
 698                             batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 699
 700                             // Profile=On
 701                             if (opt_level != 'min_opt') {
 702                                 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 703                             }
 704                         }
 705                     }
 706
 707                     def archiveSettings = new ArchivalSettings()
 708                     archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 709                     archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 710                     archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 711                     archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 712                     archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 713                     archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 714                     archiveSettings.addFiles('machinedata.json')
 715                     archiveSettings.setAlwaysArchive()
 716
 717                     Utilities.addArchival(newJob, archiveSettings)
 718                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 719
 720                     newJob.with {
 721                         logRotator {
 722                             artifactDaysToKeep(14)
 723                             daysToKeep(30)
 724                             artifactNumToKeep(100)
 725                             numToKeep(200)
 726                         }
 727                         wrappers {
 728                             timeout {
 729                                 absolute(240)
 730                             }
 731                         }
 732                     }
 733
 734                     if (isPR) {
 735                         def opts = ""
 736                         if (opt_level == 'min_opt') {
 737                             opts = '\\W+min_opts'
 738                         }
 739                         def jitt = ""
 740                         if (jit != 'ryujit') {
 741                             jitt = "\\W+${jit}"
 742                         }
 743
 744                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 745                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 746                         builder.triggerOnlyOnComment()
 747                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 748                         builder.triggerForBranch(branch)
 749                         builder.emitTrigger(newJob)
 750                     }
 751                     else if (opt_level == 'full_opt') {
 752                         // Set a push trigger
 753                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 754                         builder.emitTrigger(newJob)
 755                     }
 756                     else {
 757                         // Set periodic trigger
 758                         Utilities.addPeriodicTrigger(newJob, '@daily')
 759                     }
 760                 }
 761             }
 762         }
 763     }
 764 }
 765
 766 // Setup size-on-disk test
 767 ['Windows_NT'].each { os ->
 768     ['x64', 'x86'].each { arch ->
 769         def architecture = arch
 770         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 771             label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
 772
 773             wrappers {
 774                 credentialsBinding {
 775                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 776                 }
 777             }
 778
 779             def channel = 'master'
 780             def configuration = 'Release'
 781             def runType = 'rolling'
 782             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 783             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 784             def coreRoot = "${testBin}\\Tests\\Core_Root"
 785             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 786             def python = "C:\\python3.7.0\\python.exe"
 787
 788             steps {
 789                 // Install nuget and get BenchView tools
 790                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 791                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 792                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 793
 794                 // Generate submission metadata for BenchView
 795                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 796                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 797                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 798                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 799                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 800                 "${python} \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 801                 "${python} \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 802
 803                 // Generate machine data from BenchView
 804                 batchFile("${python} \"${benchViewTools}\\machinedata.py\"")
 805
 806                 // Build CoreCLR and gnerate test layout
 807                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 808                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 809
 810                 // Run the size on disk benchmark
 811                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 812
 813                 // From sodbench.csv, create measurment.json, then submission.json
 814                 batchFile("${python} \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 815                 batchFile("${python} \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 816
 817                 // If this is a PR, upload submission.json
 818                 batchFile("${python} \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 819             }
 820         }
 821
 822         def archiveSettings = new ArchivalSettings()
 823         archiveSettings.addFiles('bin/toArchive/**')
 824         archiveSettings.addFiles('machinedata.json')
 825         archiveSettings.setAlwaysArchive()
 826
 827         Utilities.addArchival(newJob, archiveSettings)
 828         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 829
 830         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 831         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 832
 833         newJob.with {
 834             logRotator {
 835                 artifactDaysToKeep(14)
 836                 daysToKeep(30)
 837                 artifactNumToKeep(100)
 838                 numToKeep(200)
 839             }
 840             wrappers {
 841                 timeout {
 842                     absolute(240)
 843                 }
 844             }
 845         }
 846     }
 847 }
 848
 849 // Setup IlLink tests
 850 [true, false].each { isPR ->
 851     ['Windows_NT'].each { os ->
 852         ['x64'].each { arch ->
 853             ['ryujit'].each { jit ->
 854                 ['full_opt'].each { opt_level ->
 855                     def architecture = arch
 856                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 857                         label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
 858
 859                         def testEnv = ""
 860                         def python = "C:\\python3.7.0\\python.exe"
 861                         wrappers {
 862                             credentialsBinding {
 863                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 864                             }
 865                         }
 866
 867                         if (isPR) {
 868                             parameters {
 869                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 870                             }
 871                         }
 872
 873                         parameters {
 874                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 875                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 876                         }
 877
 878                         def configuration = 'Release'
 879                         def runType = isPR ? 'private' : 'rolling'
 880                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 881                         def uploadString = '-uploadToBenchview'
 882
 883                         steps {
 884                             // Batch
 885                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 886                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 887                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 888
 889                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 890                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 891                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 892                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 893                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 894                             "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 895                             "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 896                             batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 897                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 898
 899                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 900
 901                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 902
 903                             // Scenario: ILLink
 904                             batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
 905                             "${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 906                         }
 907                     }
 908
 909                     def archiveSettings = new ArchivalSettings()
 910                     archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
 911                     archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
 912                     archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
 913                     archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
 914                     archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
 915                     archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
 916                     archiveSettings.addFiles('machinedata.json')
 917                     archiveSettings.setAlwaysArchive()
 918
 919                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 920                     Utilities.addArchival(newJob, archiveSettings)
 921                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 922
 923                     newJob.with {
 924                         logRotator {
 925                             artifactDaysToKeep(14)
 926                             daysToKeep(30)
 927                             artifactNumToKeep(100)
 928                             numToKeep(200)
 929                         }
 930                         wrappers {
 931                             timeout {
 932                                 absolute(240)
 933                             }
 934                         }
 935                     }
 936
 937                     if (isPR) {
 938                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 939                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 940                         builder.triggerOnlyOnComment()
 941                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 942                         builder.triggerForBranch(branch)
 943                         builder.emitTrigger(newJob)
 944                     }
 945                     else {
 946                         // Set a push trigger
 947                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 948                         builder.emitTrigger(newJob)
 949                     }
 950                 }
 951             }
 952         }
 953     }
 954 }
 955
 956 Utilities.createHelperJob(this, project, branch,
 957     "Welcome to the ${project} Perf help",
 958     "Have a nice day!")