perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             ['ryujit'].each { jit ->
  33                 ['full_opt', 'min_opt'].each { opt_level ->
  34
  35                     def architecture = arch
  36                     def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  37                     def testEnv = ""
  38
  39                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  40                         // Set the label.
  41                         label('windows_server_2016_clr_perf')
  42                         wrappers {
  43                             credentialsBinding {
  44                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  45                             }
  46                         }
  47
  48                         if (isPR) {
  49                             parameters {
  50                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  51                             }
  52                         }
  53
  54                         parameters {
  55                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  56                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  57                         }
  58
  59                         def configuration = 'Release'
  60                         def runType = isPR ? 'private' : 'rolling'
  61                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  62                         def uploadString = '-uploadToBenchview'
  63
  64                         steps {
  65                             // Batch
  66
  67                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  68                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  69                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  70                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  71                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  72                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  73                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  74                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  75                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  76                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  77                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  78                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  79
  80                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  81
  82                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
  83
  84                             // Run with just stopwatch: Profile=Off
  85                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  86                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  87
  88                             // Run with the full set of counters enabled: Profile=On
  89                             if (opt_level != 'min_opt') {
  90                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  91                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  92                             }
  93                         }
  94                     }
  95
  96                     def archiveSettings = new ArchivalSettings()
  97                     archiveSettings.addFiles('bin/sandbox_logs/**')
  98                     archiveSettings.addFiles('machinedata.json')
  99
 100                     Utilities.addArchival(newJob, archiveSettings)
 101                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 102
 103                     newJob.with {
 104                         logRotator {
 105                             artifactDaysToKeep(30)
 106                             daysToKeep(30)
 107                             artifactNumToKeep(200)
 108                             numToKeep(200)
 109                         }
 110                         wrappers {
 111                             timeout {
 112                                 absolute(240)
 113                             }
 114                         }
 115                     }
 116
 117                     if (isPR) {
 118                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 119                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 120
 121                         def opts = ""
 122                         if (opt_level == 'min_opt') {
 123                             opts = '\\W+min_opts'
 124                         }
 125                         def jitt = ""
 126                         if (jit != 'ryujit') {
 127                             jitt = "\\W+${jit}"
 128                         }
 129
 130                         builder.triggerOnlyOnComment()
 131                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 132                         builder.triggerForBranch(branch)
 133                         builder.emitTrigger(newJob)
 134                     }
 135                     else {
 136                         // Set a push trigger
 137                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 138                         builder.emitTrigger(newJob)
 139                     }
 140                 }
 141             }
 142         }
 143     }
 144 }
 145
 146 // Setup throughput perflab tests runs
 147 [true, false].each { isPR ->
 148     ['Windows_NT'].each { os ->
 149         ['x64', 'x86'].each { arch ->
 150             ['ryujit'].each { jit ->
 151                 [true, false].each { pgo_optimized ->
 152                     ['full_opt', 'min_opt'].each { opt_level ->
 153                         def architecture = arch
 154
 155                         pgo_build = ""
 156                         pgo_test = ""
 157                         pgo_string = "pgo"
 158                         if (!pgo_optimized) {
 159                             pgo_build = " -nopgooptimize"
 160                             pgo_test = " -nopgo"
 161                             pgo_string = "nopgo"
 162                         }
 163
 164                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 165                             // Set the label.
 166                             label('windows_server_2016_clr_perf')
 167                             wrappers {
 168                                 credentialsBinding {
 169                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 170                                 }
 171                             }
 172
 173                             if (isPR) {
 174                                 parameters {
 175                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 176                                 }
 177                             }
 178
 179                             def configuration = 'Release'
 180                             def runType = isPR ? 'private' : 'rolling'
 181                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 182
 183                             steps {
 184                                 // Batch
 185                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 186                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 187                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 188                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 189                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 190                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 191                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 192                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 193                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 194                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 195                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 196                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 197                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 198                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 199                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 200                             }
 201                         }
 202
 203                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 204                         def archiveSettings = new ArchivalSettings()
 205                         archiveSettings.addFiles('throughput-*.csv')
 206                         Utilities.addArchival(newJob, archiveSettings)
 207
 208                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 209
 210                         if (isPR) {
 211                             def opts = ""
 212                             if (opt_level == 'min_opt') {
 213                                 opts = '\\W+min_opts'
 214                             }
 215
 216                             def jitt = ""
 217                             if (jit != 'ryujit') {
 218                                 jitt = "\\W+${jit}"
 219                             }
 220
 221                             def pgo_trigger = ""
 222                             if (pgo_optimized) {
 223                                 pgo_trigger = "\\W+nopgo"
 224                             }
 225
 226
 227                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 228                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 229                             builder.triggerOnlyOnComment()
 230                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 231                             builder.triggerForBranch(branch)
 232                             builder.emitTrigger(newJob)
 233                         }
 234                         else {
 235                             // Set a push trigger
 236                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 237                             builder.emitTrigger(newJob)
 238                         }
 239                     }
 240                 }
 241             }
 242         }
 243     }
 244 }
 245
 246 def static getFullPerfJobName(def project, def os, def isPR) {
 247     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 248 }
 249
 250 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 251 [true, false].each { isPR ->
 252     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 253     def architecture = 'x64'
 254     def configuration = 'Release'
 255
 256     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 257     ['RHEL7.2'].each { os ->
 258         def newBuildJob = job(fullBuildJobName) {
 259             steps {
 260                 shell("./build.sh verbose ${architecture} ${configuration}")
 261             }
 262         }
 263         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 264         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 265         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 266     }
 267
 268
 269     // Actual perf testing on the following OSes
 270     def perfOSList = ['Ubuntu16.04']
 271     perfOSList.each { os ->
 272         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 273
 274             label('ubuntu_1604_clr_perf')
 275             wrappers {
 276                 credentialsBinding {
 277                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 278                 }
 279             }
 280
 281             if (isPR) {
 282                 parameters {
 283                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 284                 }
 285             }
 286
 287             parameters {
 288                 // Cap the maximum number of iterations to 21.
 289                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 290                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 291                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 292             }
 293
 294             def osGroup = getOSGroup(os)
 295             def runType = isPR ? 'private' : 'rolling'
 296             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 297
 298             steps {
 299                 shell("./tests/scripts/perf-prep.sh")
 300                 shell("./init-tools.sh")
 301                 copyArtifacts(fullBuildJobName) {
 302                     includePatterns("bin/**")
 303                     buildSelector {
 304                         buildNumber('\${PRODUCT_BUILD}')
 305                     }
 306                 }
 307                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 308                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 309                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 310                 shell("""./tests/scripts/run-xunit-perf.sh \\
 311                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 312                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 313                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 314                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 315                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 316                 --runType=\"${runType}\" \\
 317                 --benchViewOS=\"${os}\" \\
 318                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 319                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 320                 --uploadToBenchview""")
 321                 shell("mkdir -p bin/toArchive/sandbox/Logs/")
 322                 shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
 323             }
 324         }
 325
 326         def archiveSettings = new ArchivalSettings()
 327         archiveSettings.addFiles('bin/toArchive/**')
 328         archiveSettings.addFiles('machinedata.json')
 329
 330         Utilities.addArchival(newJob, archiveSettings)
 331         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 332
 333         // For perf, we need to keep the run results longer
 334         newJob.with {
 335             // Enable the log rotator
 336             logRotator {
 337                 artifactDaysToKeep(30)
 338                 daysToKeep(30)
 339                 artifactNumToKeep(200)
 340                 numToKeep(200)
 341             }
 342             wrappers {
 343                 timeout {
 344                     absolute(240)
 345                 }
 346             }
 347         }
 348     } // os
 349
 350     def flowJobPerfRunList = perfOSList.collect { os ->
 351         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 352     }
 353     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 354         if (isPR) {
 355             parameters {
 356                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 357             }
 358         }
 359         buildFlow("""
 360 // First, build the bits on RHEL7.2
 361 b = build(params, '${fullBuildJobName}')
 362
 363 // Then, run the perf tests
 364 parallel(
 365     ${flowJobPerfRunList.join(",\n    ")}
 366 )
 367 """)
 368     }
 369
 370     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 371     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 372
 373     if (isPR) {
 374         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 375         builder.setGithubContext("Linux Perf Test Flow")
 376         builder.triggerOnlyOnComment()
 377         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 378         builder.triggerForBranch(branch)
 379         builder.emitTrigger(newFlowJob)
 380     }
 381     else {
 382         // Set a push trigger
 383         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 384         builder.emitTrigger(newFlowJob)
 385     }
 386
 387 } // isPR
 388
 389 def static getFullThroughputJobName(def project, def os, def isPR) {
 390     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 391 }
 392
 393 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 394 [true, false].each { isPR ->
 395     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 396     def architecture = 'x64'
 397     def configuration = 'Release'
 398
 399     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 400     ['RHEL7.2'].each { os ->
 401         def newBuildJob = job(fullBuildJobName) {
 402             steps {
 403                 shell("./build.sh verbose ${architecture} ${configuration}")
 404             }
 405         }
 406         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 407         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 408         Utilities.addArchival(newBuildJob, "bin/Product/**")
 409     }
 410
 411     // Actual perf testing on the following OSes
 412     def throughputOSList = ['Ubuntu14.04']
 413     def throughputOptLevelList = ['full_opt', 'min_opt']
 414
 415     def throughputOSOptLevelList = []
 416
 417     throughputOSList.each { os ->
 418         throughputOptLevelList.each { opt_level ->
 419             throughputOSOptLevelList.add("${os}_${opt_level}")
 420         }
 421     }
 422
 423     throughputOSList.each { os ->
 424         throughputOptLevelList.each { opt_level ->
 425             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 426
 427                 label('ubuntu_1604_clr_perf')
 428                     wrappers {
 429                         credentialsBinding {
 430                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 431                         }
 432                     }
 433
 434                 if (isPR) {
 435                     parameters {
 436                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 437                     }
 438                 }
 439
 440                 parameters {
 441                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 442                 }
 443
 444                 def osGroup = getOSGroup(os)
 445                 def runType = isPR ? 'private' : 'rolling'
 446                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 447
 448                 steps {
 449                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 450                     shell("./init-tools.sh")
 451                     copyArtifacts(fullBuildJobName) {
 452                         includePatterns("bin/Product/**")
 453                         buildSelector {
 454                             buildNumber('\${PRODUCT_BUILD}')
 455                         }
 456                     }
 457                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 458                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 459                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 460                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 461                     -arch \"${architecture}\" \\
 462                     -os \"${os}\" \\
 463                     -configuration \"${configuration}\" \\
 464                     -opt_level \"${opt_level}\" \\
 465                     -clr_root \"\${WORKSPACE}\" \\
 466                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 467                     -run_type \"${runType}\" \\
 468                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 469                 }
 470             }
 471
 472             // Save machinedata.json to /artifact/bin/ Jenkins dir
 473             def archiveSettings = new ArchivalSettings()
 474             archiveSettings.addFiles('throughput-*.csv')
 475             archiveSettings.addFiles('machinedata.json')
 476             Utilities.addArchival(newJob, archiveSettings)
 477
 478             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 479
 480             // For perf, we need to keep the run results longer
 481             newJob.with {
 482                 // Enable the log rotator
 483                 logRotator {
 484                     artifactDaysToKeep(7)
 485                     daysToKeep(300)
 486                     artifactNumToKeep(25)
 487                     numToKeep(1000)
 488                 }
 489             }
 490         } // opt_level
 491     } // os
 492
 493     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 494         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 495     }
 496     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 497         if (isPR) {
 498             parameters {
 499                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 500             }
 501         }
 502         buildFlow("""
 503 // First, build the bits on RHEL7.2
 504 b = build(params, '${fullBuildJobName}')
 505
 506 // Then, run the perf tests
 507 parallel(
 508     ${flowJobTPRunList.join(",\n    ")}
 509 )
 510 """)
 511     }
 512
 513     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 514     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 515
 516     if (isPR) {
 517         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 518         builder.setGithubContext("Linux Throughput Perf Test Flow")
 519         builder.triggerOnlyOnComment()
 520         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 521         builder.triggerForBranch(branch)
 522         builder.emitTrigger(newFlowJob)
 523     }
 524     else {
 525         // Set a push trigger
 526         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 527         builder.emitTrigger(newFlowJob)
 528     }
 529
 530 } // isPR
 531
 532 // Setup CoreCLR-Scenarios tests
 533 [true, false].each { isPR ->
 534     ['Windows_NT'].each { os ->
 535         ['x64', 'x86'].each { arch ->
 536             ['ryujit'].each { jit ->
 537                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 538                     def architecture = arch
 539                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 540
 541                         def testEnv = ""
 542
 543                         // Set the label.
 544                         label('windows_server_2016_clr_perf')
 545                         wrappers {
 546                             credentialsBinding {
 547                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 548                             }
 549                         }
 550
 551                         if (isPR) {
 552                             parameters {
 553                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 554                             }
 555                         }
 556
 557                         parameters {
 558                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 559                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 560                         }
 561
 562                         def configuration = 'Release'
 563                         def runType = isPR ? 'private' : 'rolling'
 564                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 565                         def uploadString = '-uploadToBenchview'
 566
 567                         steps {
 568                             // Batch
 569                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 570                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 571                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 572
 573                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 574                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 575                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 576                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 577                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 578                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 579                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 580                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 581                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 582
 583                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 584
 585                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
 586
 587                             // Profile=Off
 588                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 589
 590                             // Profile=On
 591                             if (opt_level != 'min_opt') {
 592                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 593                             }
 594                         }
 595                     }
 596
 597                     def archiveSettings = new ArchivalSettings()
 598                     archiveSettings.addFiles('bin/sandbox_logs/**')
 599                     archiveSettings.addFiles('machinedata.json')
 600
 601                     Utilities.addArchival(newJob, archiveSettings)
 602                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 603
 604                     newJob.with {
 605                         logRotator {
 606                             artifactDaysToKeep(30)
 607                             daysToKeep(30)
 608                             artifactNumToKeep(200)
 609                             numToKeep(200)
 610                         }
 611                         wrappers {
 612                             timeout {
 613                                 absolute(240)
 614                             }
 615                         }
 616                     }
 617
 618                     if (isPR) {
 619                         def opts = ""
 620                         if (opt_level == 'min_opt') {
 621                             opts = '\\W+min_opts'
 622                         }
 623                         def jitt = ""
 624                         if (jit != 'ryujit') {
 625                             jitt = "\\W+${jit}"
 626                         }
 627
 628                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 629                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 630                         builder.triggerOnlyOnComment()
 631                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 632                         builder.triggerForBranch(branch)
 633                         builder.emitTrigger(newJob)
 634                     }
 635                     else {
 636                         // Set a push trigger
 637                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 638                         builder.emitTrigger(newJob)
 639                     }
 640                 }
 641             }
 642         }
 643     }
 644 }
 645
 646 // Setup size-on-disk test
 647 ['Windows_NT'].each { os ->
 648     ['x64', 'x86'].each { arch ->
 649         def architecture = arch
 650         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 651
 652             wrappers {
 653                 credentialsBinding {
 654                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 655                 }
 656             }
 657
 658             def channel = 'master'
 659             def configuration = 'Release'
 660             def runType = 'rolling'
 661             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 662             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 663             def coreRoot = "${testBin}\\Tests\\Core_Root"
 664             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 665
 666             steps {
 667                 // Install nuget and get BenchView tools
 668                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 669                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 670                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 671
 672                 // Generate submission metadata for BenchView
 673                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 674                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 675                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 676                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 677                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 678                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 679                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 680
 681                 // Generate machine data from BenchView
 682                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 683
 684                 // Build CoreCLR and gnerate test layout
 685                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 686                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 687
 688                 // Run the size on disk benchmark
 689                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 690
 691                 // From sodbench.csv, create measurment.json, then submission.json
 692                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 693                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 694
 695                 // If this is a PR, upload submission.json
 696                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 697             }
 698         }
 699
 700         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 701
 702         def archiveSettings = new ArchivalSettings()
 703         archiveSettings.addFiles('bin/toArchive/**')
 704         archiveSettings.addFiles('machinedata.json')
 705
 706         Utilities.addArchival(newJob, archiveSettings)
 707         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 708
 709         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 710         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 711
 712         newJob.with {
 713             logRotator {
 714                 artifactDaysToKeep(30)
 715                 daysToKeep(30)
 716                 artifactNumToKeep(200)
 717                 numToKeep(200)
 718             }
 719             wrappers {
 720                 timeout {
 721                     absolute(240)
 722                 }
 723             }
 724         }
 725     }
 726 }
 727
 728 // Setup IlLink tests
 729 [true, false].each { isPR ->
 730     ['Windows_NT'].each { os ->
 731         ['x64'].each { arch ->
 732             ['ryujit'].each { jit ->
 733                 ['full_opt'].each { opt_level ->
 734                     def architecture = arch
 735                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 736
 737                         def testEnv = ""
 738                         wrappers {
 739                             credentialsBinding {
 740                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 741                             }
 742                         }
 743
 744                         if (isPR) {
 745                             parameters {
 746                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 747                             }
 748                         }
 749
 750                         parameters {
 751                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 752                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 753                         }
 754
 755                         def configuration = 'Release'
 756                         def runType = isPR ? 'private' : 'rolling'
 757                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 758                         def uploadString = '-uploadToBenchview'
 759
 760                         steps {
 761                             // Batch
 762                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 763                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 764                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 765
 766                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 767                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 768                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 769                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 770                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 771                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 772                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 773                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 774                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 775
 776                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 777
 778                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 779
 780                             // Scenario: ILLink
 781                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 782                         }
 783                     }
 784
 785                     def archiveSettings = new ArchivalSettings()
 786                     archiveSettings.addFiles('bin/sandbox_logs/**')
 787                     archiveSettings.addFiles('machinedata.json')
 788
 789                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 790                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 791                     Utilities.addArchival(newJob, archiveSettings)
 792                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 793
 794                     newJob.with {
 795                         logRotator {
 796                             artifactDaysToKeep(30)
 797                             daysToKeep(30)
 798                             artifactNumToKeep(200)
 799                             numToKeep(200)
 800                         }
 801                         wrappers {
 802                             timeout {
 803                                 absolute(240)
 804                             }
 805                         }
 806                     }
 807
 808                     if (isPR) {
 809                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 810                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 811                         builder.triggerOnlyOnComment()
 812                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 813                         builder.triggerForBranch(branch)
 814                         builder.emitTrigger(newJob)
 815                     }
 816                     else {
 817                         // Set a push trigger
 818                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 819                         builder.emitTrigger(newJob)
 820                     }
 821                 }
 822             }
 823         }
 824     }
 825 }
 826
 827 Utilities.createHelperJob(this, project, branch,
 828     "Welcome to the ${project} Perf help",
 829     "Have a nice day!")