perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 def architecture = arch
  34                 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_smoketest" : "perf_perflab_${os}_${arch}"
  35
  36                 if (arch == 'x86') {
  37                     testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  38                 }
  39
  40                 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  41                     // Set the label.
  42                     label('windows_clr_perf')
  43                     wrappers {
  44                         credentialsBinding {
  45                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  46                         }
  47                     }
  48
  49                     if (isPR) {
  50                         parameters {
  51                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  52                         }
  53                     }
  54
  55                     if (isSmoketest) {
  56                         parameters {
  57                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  58                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  59                         }
  60                     }
  61                     else {
  62                         parameters {
  63                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  64                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  65                         }
  66                     }
  67
  68                     def configuration = 'Release'
  69                     def runType = isPR ? 'private' : 'rolling'
  70                     def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  71                     def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  72
  73                     steps {
  74                         // Batch
  75
  76                         batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  77                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  78                         batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  79                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  80                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  81                         batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  82                         "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  83                         "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
  84                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
  85                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  86                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  87                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  88
  89                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  90
  91                         def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
  92
  93                         // Run with just stopwatch: Profile=Off
  94                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  95                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  96
  97                         // Run with the full set of counters enabled: Profile=On
  98                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  99                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 100                     }
 101                 }
 102
 103                 if (isSmoketest) {
 104                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 105                 }
 106
 107                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 108                 def archiveSettings = new ArchivalSettings()
 109                 archiveSettings.addFiles('bin/sandbox/Logs/Perf-*.*')
 110                 archiveSettings.addFiles('machinedata.json')
 111                 Utilities.addArchival(newJob, archiveSettings)
 112
 113                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 114
 115                 newJob.with {
 116                     wrappers {
 117                         timeout {
 118                             absolute(240)
 119                         }
 120                     }
 121                 }
 122
 123                 if (isPR) {
 124                     TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 125                     if (isSmoketest)
 126                     {
 127                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests Correctness")
 128                     }
 129                     else
 130                     {
 131                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 132                         builder.triggerOnlyOnComment()
 133                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 134                     }
 135                     builder.triggerForBranch(branch)
 136                     builder.emitTrigger(newJob)
 137                 }
 138                 else {
 139                     // Set a push trigger
 140                     TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 141                     builder.emitTrigger(newJob)
 142                 }
 143             }
 144         }
 145     }
 146 }
 147
 148 // Setup throughput perflab tests runs
 149 [true, false].each { isPR ->
 150     ['Windows_NT'].each { os ->
 151         ['x64', 'x86'].each { arch ->
 152             ['full_opt', 'min_opt'].each { opt_level ->
 153                 def architecture = arch
 154
 155                 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}", isPR)) {
 156                     // Set the label.
 157                     label('windows_clr_perf')
 158                     wrappers {
 159                         credentialsBinding {
 160                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 161                         }
 162                     }
 163
 164                     if (isPR) {
 165                         parameters {
 166                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 167                         }
 168                     }
 169
 170                     def configuration = 'Release'
 171                     def runType = isPR ? 'private' : 'rolling'
 172                     def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 173
 174                     steps {
 175                         // Batch
 176                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 177                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 178                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 179                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 180                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 181                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 182                         batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 183                         "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 184                         "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 185                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 186                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 187                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 188                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 189                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 190                         batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 191                     }
 192                 }
 193
 194                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 195                 def archiveSettings = new ArchivalSettings()
 196                 archiveSettings.addFiles('throughput-*.csv')
 197                 Utilities.addArchival(newJob, archiveSettings)
 198
 199                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 200
 201                 if (isPR) {
 202                     def opts = ""
 203                     if (opt_level == 'min_opts')
 204                     {
 205                         opts = '\\W+min_opts'
 206                     }
 207                     TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 208                     builder.setGithubContext("${os} ${arch} ${opt_level} CoreCLR Throughput Perf Tests")
 209                     builder.triggerOnlyOnComment()
 210                     builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}\\W+throughput.*")
 211                     builder.triggerForBranch(branch)
 212                     builder.emitTrigger(newJob)
 213                 }
 214                 else {
 215                     // Set a push trigger
 216                     TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 217                     builder.emitTrigger(newJob)
 218                 }
 219             }
 220         }
 221     }
 222 }
 223
 224 def static getFullPerfJobName(def project, def os, def isPR) {
 225     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 226 }
 227
 228 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 229 [true, false].each { isPR ->
 230     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 231     def architecture = 'x64'
 232     def configuration = 'Release'
 233
 234     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 235     ['RHEL7.2'].each { os ->
 236         def newBuildJob = job(fullBuildJobName) {
 237             steps {
 238                 shell("./build.sh verbose ${architecture} ${configuration}")
 239             }
 240         }
 241         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 242         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 243         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 244     }
 245
 246
 247     // Actual perf testing on the following OSes
 248     def perfOSList = ['Ubuntu14.04']
 249     perfOSList.each { os ->
 250         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 251
 252             label('linux_clr_perf')
 253             wrappers {
 254                 credentialsBinding {
 255                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 256                 }
 257             }
 258
 259             if (isPR) {
 260                 parameters {
 261                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 262                 }
 263             }
 264
 265             parameters {
 266                 // Cap the maximum number of iterations to 21.
 267                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
 268                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
 269                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 270             }
 271
 272             def osGroup = getOSGroup(os)
 273             def runType = isPR ? 'private' : 'rolling'
 274             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 275
 276             steps {
 277                 shell("./tests/scripts/perf-prep.sh")
 278                 shell("./init-tools.sh")
 279                 copyArtifacts(fullBuildJobName) {
 280                     includePatterns("bin/**")
 281                     buildSelector {
 282                         buildNumber('\${PRODUCT_BUILD}')
 283                     }
 284                 }
 285                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 286                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 287                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 288                 shell("""./tests/scripts/run-xunit-perf.sh \\
 289                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 290                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 291                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 292                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 293                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 294                 --runType=\"${runType}\" \\
 295                 --benchViewOS=\"${os}\" \\
 296                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 297                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 298                 --uploadToBenchview""")
 299             }
 300         }
 301
 302         // Save machinedata.json to /artifact/bin/ Jenkins dir
 303         def archiveSettings = new ArchivalSettings()
 304         archiveSettings.addFiles('bin/sandbox/Logs/Perf-*.*')
 305         archiveSettings.addFiles('machinedata.json')
 306         Utilities.addArchival(newJob, archiveSettings)
 307
 308         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 309
 310         // For perf, we need to keep the run results longer
 311         newJob.with {
 312             // Enable the log rotator
 313             logRotator {
 314                 artifactDaysToKeep(7)
 315                 daysToKeep(300)
 316                 artifactNumToKeep(25)
 317                 numToKeep(1000)
 318             }
 319         }
 320     } // os
 321
 322     def flowJobPerfRunList = perfOSList.collect { os ->
 323         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 324     }
 325     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 326         if (isPR) {
 327             parameters {
 328                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 329             }
 330         }
 331         buildFlow("""
 332 // First, build the bits on RHEL7.2
 333 b = build(params, '${fullBuildJobName}')
 334
 335 // Then, run the perf tests
 336 parallel(
 337     ${flowJobPerfRunList.join(",\n    ")}
 338 )
 339 """)
 340     }
 341
 342     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 343     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 344
 345     if (isPR) {
 346         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 347         builder.setGithubContext("Linux Perf Test Flow")
 348         builder.triggerOnlyOnComment()
 349         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 350         builder.triggerForBranch(branch)
 351         builder.emitTrigger(newFlowJob)
 352     }
 353     else {
 354         // Set a push trigger
 355         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 356         builder.emitTrigger(newFlowJob)
 357     }
 358
 359 } // isPR
 360
 361 def static getFullThroughputJobName(def project, def os, def isPR) {
 362     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 363 }
 364
 365 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 366 [true, false].each { isPR ->
 367     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 368     def architecture = 'x64'
 369     def configuration = 'Release'
 370
 371     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 372     ['RHEL7.2'].each { os ->
 373         def newBuildJob = job(fullBuildJobName) {
 374             steps {
 375                 shell("./build.sh verbose ${architecture} ${configuration}")
 376             }
 377         }
 378         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 379         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 380         Utilities.addArchival(newBuildJob, "bin/Product/**")
 381     }
 382
 383     // Actual perf testing on the following OSes
 384     def throughputOSList = ['Ubuntu14.04']
 385     def throughputOptLevelList = ['full_opt', 'min_opt']
 386
 387     def throughputOSOptLevelList = []
 388
 389     throughputOSList.each { os ->
 390         throughputOptLevelList.each { opt_level ->
 391             throughputOSOptLevelList.add("${os}_${opt_level}")
 392         }
 393     }
 394
 395     throughputOSList.each { os ->
 396         throughputOptLevelList.each { opt_level ->
 397             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 398
 399                 label('linux_clr_perf')
 400                     wrappers {
 401                         credentialsBinding {
 402                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 403                         }
 404                     }
 405
 406                 if (isPR)
 407                 {
 408                     parameters
 409                     {
 410                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 411                     }
 412                 }
 413
 414                 parameters {
 415                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 416                 }
 417
 418                 def osGroup = getOSGroup(os)
 419                 def runType = isPR ? 'private' : 'rolling'
 420                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 421
 422                 steps {
 423                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 424                     shell("./init-tools.sh")
 425                     copyArtifacts(fullBuildJobName) {
 426                         includePatterns("bin/Product/**")
 427                         buildSelector {
 428                             buildNumber('\${PRODUCT_BUILD}')
 429                         }
 430                     }
 431                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 432                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 433                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 434                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 435                     -arch \"${architecture}\" \\
 436                     -os \"${os}\" \\
 437                     -configuration \"${configuration}\" \\
 438                     -opt_level \"${opt_level}\" \\
 439                     -clr_root \"\${WORKSPACE}\" \\
 440                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 441                     -run_type \"${runType}\" \\
 442                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 443                 }
 444             }
 445
 446             // Save machinedata.json to /artifact/bin/ Jenkins dir
 447             def archiveSettings = new ArchivalSettings()
 448             archiveSettings.addFiles('throughput-*.csv')
 449             archiveSettings.addFiles('machinedata.json')
 450             Utilities.addArchival(newJob, archiveSettings)
 451
 452             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 453
 454             // For perf, we need to keep the run results longer
 455             newJob.with {
 456                 // Enable the log rotator
 457                 logRotator {
 458                     artifactDaysToKeep(7)
 459                     daysToKeep(300)
 460                     artifactNumToKeep(25)
 461                     numToKeep(1000)
 462                 }
 463             }
 464         } // opt_level
 465     } // os
 466
 467     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 468         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 469     }
 470     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 471         if (isPR) {
 472             parameters {
 473                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 474             }
 475         }
 476         buildFlow("""
 477 // First, build the bits on RHEL7.2
 478 b = build(params, '${fullBuildJobName}')
 479
 480 // Then, run the perf tests
 481 parallel(
 482     ${flowJobTPRunList.join(",\n    ")}
 483 )
 484 """)
 485     }
 486
 487     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 488     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 489
 490     if (isPR) {
 491         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 492         builder.setGithubContext("Linux Throughput Perf Test Flow")
 493         builder.triggerOnlyOnComment()
 494         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 495         builder.triggerForBranch(branch)
 496         builder.emitTrigger(newFlowJob)
 497     }
 498     else {
 499         // Set a push trigger
 500         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 501         builder.emitTrigger(newFlowJob)
 502     }
 503
 504 } // isPR
 505
 506 // Setup CoreCLR-Scenarios tests
 507 [true, false].each { isPR ->
 508     ['Windows_NT'].each { os ->
 509         ['x64', 'x86'].each { arch ->
 510             def architecture = arch
 511             def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}", isPR)) {
 512                 // Set the label.
 513                 label('windows_clr_perf')
 514                 wrappers {
 515                     credentialsBinding {
 516                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 517                     }
 518                 }
 519
 520                 if (isPR) {
 521                     parameters {
 522                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 523                     }
 524                 }
 525
 526                 parameters {
 527                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 528                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 529                 }
 530
 531                 def configuration = 'Release'
 532                 def runType = isPR ? 'private' : 'rolling'
 533                 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 534                 def uploadString = '-uploadToBenchview'
 535
 536                 steps {
 537                     // Batch
 538                     batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 539                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 540                     batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 541
 542                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 543                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 544                     batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 545                     "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 546                     "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 547                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
 548                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 549                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 550                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 551
 552                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 553
 554                     def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -scenarioTest"
 555                     def failedOutputLogFilename = "run-xunit-perf-scenario.log"
 556
 557                     // Using a sentinel file to
 558                     batchFile("if exist \"${failedOutputLogFilename}\" del /q /f \"${failedOutputLogFilename}\"")
 559                     batchFile("if exist \"${failedOutputLogFilename}\" (echo [ERROR] Failed to delete previously created \"${failedOutputLogFilename}\" file.& exit /b 1)")
 560
 561                     // Scenario: JitBench
 562                     batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios || (echo [ERROR] JitBench failed. 1>>\"${failedOutputLogFilename}\"& exit /b 0)")
 563
 564                     // Scenario: ILLink
 565                     if (arch == 'x64') {
 566                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup || (echo [ERROR] IlLink failed. 1>>\"${failedOutputLogFilename}\"& exit /b 0)")
 567                     }
 568
 569                     batchFile("if exist \"${failedOutputLogFilename}\" (type \"${failedOutputLogFilename}\"& exit /b 1)")
 570                 }
 571             }
 572
 573              // Save machinedata.json to /artifact/bin/ Jenkins dir
 574             def archiveSettings = new ArchivalSettings()
 575             archiveSettings.addFiles('bin/sandbox/Perf-*.*')
 576             archiveSettings.addFiles('machinedata.json')
 577             Utilities.addArchival(newJob, archiveSettings)
 578
 579             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 580
 581             newJob.with {
 582                 wrappers {
 583                     timeout {
 584                         absolute(240)
 585                     }
 586                 }
 587             }
 588
 589             if (isPR) {
 590                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 591                 builder.setGithubContext("${os} ${arch} Performance Scenarios Tests")
 592                 builder.triggerOnlyOnComment()
 593                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf\\W+scenarios.*")
 594                 builder.triggerForBranch(branch)
 595                 builder.emitTrigger(newJob)
 596             }
 597             else {
 598                 // Set a push trigger
 599                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 600                 builder.emitTrigger(newJob)
 601             }
 602         }
 603     }
 604 }
 605
 606 Utilities.createHelperJob(this, project, branch,
 607     "Welcome to the ${project} Perf help",
 608     "Have a nice day!")