perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             ['ryujit'].each { jit ->
  33                 ['full_opt', 'min_opt'].each { opt_level ->
  34
  35                     def architecture = arch
  36                     def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  37                     def testEnv = ""
  38
  39                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  40                         // Set the label.
  41                         label('windows_server_2016_clr_perf')
  42                         wrappers {
  43                             credentialsBinding {
  44                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  45                             }
  46                         }
  47
  48                         if (isPR) {
  49                             parameters {
  50                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  51                             }
  52                         }
  53
  54                         parameters {
  55                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  56                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  57                         }
  58
  59                         def configuration = 'Release'
  60                         def runType = isPR ? 'private' : 'rolling'
  61                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  62                         def uploadString = '-uploadToBenchview'
  63
  64                         steps {
  65                             // Batch
  66
  67                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  68                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  69                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  70                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  71                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  72                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  73                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  74                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  75                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  76                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  77                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  78                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  79
  80                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  81
  82                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
  83
  84                             // Run with just stopwatch: Profile=Off
  85                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  86                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  87
  88                             // Run with the full set of counters enabled: Profile=On
  89                             if (opt_level != 'min_opt') {
  90                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  91                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  92                             }
  93                         }
  94                     }
  95
  96                     def archiveSettings = new ArchivalSettings()
  97                     archiveSettings.addFiles('bin/sandbox_logs/**')
  98                     archiveSettings.addFiles('machinedata.json')
  99                     archiveSettings.setAlwaysArchive()
 100
 101                     Utilities.addArchival(newJob, archiveSettings)
 102                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 103
 104                     newJob.with {
 105                         logRotator {
 106                             artifactDaysToKeep(30)
 107                             daysToKeep(30)
 108                             artifactNumToKeep(200)
 109                             numToKeep(200)
 110                         }
 111                         wrappers {
 112                             timeout {
 113                                 absolute(240)
 114                             }
 115                         }
 116                     }
 117
 118                     if (isPR) {
 119                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 120                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 121
 122                         def opts = ""
 123                         if (opt_level == 'min_opt') {
 124                             opts = '\\W+min_opts'
 125                         }
 126                         def jitt = ""
 127                         if (jit != 'ryujit') {
 128                             jitt = "\\W+${jit}"
 129                         }
 130
 131                         builder.triggerOnlyOnComment()
 132                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 133                         builder.triggerForBranch(branch)
 134                         builder.emitTrigger(newJob)
 135                     }
 136                     else if (opt_level == 'full_opt') {
 137                         // Set a push trigger
 138                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 139                         builder.emitTrigger(newJob)
 140                     }
 141                     else {
 142                         // Set periodic trigger
 143                         Utilities.addPeriodicTrigger(newJob, '@daily')
 144                     }
 145                 }
 146             }
 147         }
 148     }
 149 }
 150
 151 // Setup throughput perflab tests runs
 152 [true, false].each { isPR ->
 153     ['Windows_NT'].each { os ->
 154         ['x64', 'x86'].each { arch ->
 155             ['ryujit'].each { jit ->
 156                 [true, false].each { pgo_optimized ->
 157                     ['full_opt', 'min_opt'].each { opt_level ->
 158                         def architecture = arch
 159
 160                         pgo_build = ""
 161                         pgo_test = ""
 162                         pgo_string = "pgo"
 163                         if (!pgo_optimized) {
 164                             pgo_build = " -nopgooptimize"
 165                             pgo_test = " -nopgo"
 166                             pgo_string = "nopgo"
 167                         }
 168
 169                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 170                             // Set the label.
 171                             label('windows_server_2016_clr_perf')
 172                             wrappers {
 173                                 credentialsBinding {
 174                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 175                                 }
 176                             }
 177
 178                             if (isPR) {
 179                                 parameters {
 180                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 181                                 }
 182                             }
 183
 184                             def configuration = 'Release'
 185                             def runType = isPR ? 'private' : 'rolling'
 186                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 187
 188                             steps {
 189                                 // Batch
 190                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 191                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 192                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 193                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 194                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 195                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 196                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 197                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 198                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 199                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 200                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 201                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 202                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 203                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 204                             }
 205                         }
 206
 207                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 208                         def archiveSettings = new ArchivalSettings()
 209                         archiveSettings.addFiles('throughput-*.csv')
 210                         archiveSettings.setAlwaysArchive()
 211                         Utilities.addArchival(newJob, archiveSettings)
 212
 213                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 214
 215                         if (isPR) {
 216                             def opts = ""
 217                             if (opt_level == 'min_opt') {
 218                                 opts = '\\W+min_opts'
 219                             }
 220
 221                             def jitt = ""
 222                             if (jit != 'ryujit') {
 223                                 jitt = "\\W+${jit}"
 224                             }
 225
 226                             def pgo_trigger = ""
 227                             if (pgo_optimized) {
 228                                 pgo_trigger = "\\W+nopgo"
 229                             }
 230
 231
 232                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 233                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 234                             builder.triggerOnlyOnComment()
 235                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 236                             builder.triggerForBranch(branch)
 237                             builder.emitTrigger(newJob)
 238                         }
 239                         else if (opt_level == 'full_opt' && pgo_optimized) {
 240                             // Set a push trigger
 241                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 242                             builder.emitTrigger(newJob)
 243                         }
 244                         else {
 245                             // Set periodic trigger
 246                             Utilities.addPeriodicTrigger(newJob, '@daily')
 247                         }
 248                     }
 249                 }
 250             }
 251         }
 252     }
 253 }
 254
 255 def static getFullPerfJobName(def project, def os, def isPR) {
 256     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 257 }
 258
 259 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 260 [true, false].each { isPR ->
 261     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 262     def architecture = 'x64'
 263     def configuration = 'Release'
 264
 265     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 266     ['RHEL7.2'].each { os ->
 267         def newBuildJob = job(fullBuildJobName) {
 268             steps {
 269                 shell("./build.sh verbose ${architecture} ${configuration}")
 270                 shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}")
 271             }
 272         }
 273         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 274         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 275         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
 276     }
 277
 278
 279     // Actual perf testing on the following OSes
 280     def perfOSList = ['Ubuntu16.04']
 281     perfOSList.each { os ->
 282         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 283
 284             label('ubuntu_1604_clr_perf')
 285             wrappers {
 286                 credentialsBinding {
 287                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 288                 }
 289             }
 290
 291             if (isPR) {
 292                 parameters {
 293                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 294                 }
 295             }
 296
 297             parameters {
 298                 // Cap the maximum number of iterations to 21.
 299                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 300                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 301                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 302             }
 303
 304             def osGroup = getOSGroup(os)
 305             def runType = isPR ? 'private' : 'rolling'
 306             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 307             def uploadString = '-uploadToBenchview'
 308
 309             def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
 310
 311             steps {
 312                 shell("./tests/scripts/perf-prep.sh --nocorefx")
 313                 shell("./init-tools.sh")
 314                 copyArtifacts(fullBuildJobName) {
 315                     includePatterns("bin/**")
 316                     buildSelector {
 317                         buildNumber('\${PRODUCT_BUILD}')
 318                     }
 319                 }
 320                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 321                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 322                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 323                 shell("""python3 ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
 324             }
 325         }
 326
 327         def archiveSettings = new ArchivalSettings()
 328         archiveSettings.addFiles('bin/sandbox_logs/**')
 329         archiveSettings.addFiles('machinedata.json')
 330         archiveSettings.setAlwaysArchive()
 331
 332         Utilities.addArchival(newJob, archiveSettings)
 333         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 334
 335         // For perf, we need to keep the run results longer
 336         newJob.with {
 337             // Enable the log rotator
 338             logRotator {
 339                 artifactDaysToKeep(30)
 340                 daysToKeep(30)
 341                 artifactNumToKeep(200)
 342                 numToKeep(200)
 343             }
 344             wrappers {
 345                 timeout {
 346                     absolute(240)
 347                 }
 348             }
 349         }
 350     } // os
 351
 352     def flowJobPerfRunList = perfOSList.collect { os ->
 353         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 354     }
 355     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 356         if (isPR) {
 357             parameters {
 358                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 359             }
 360         }
 361         buildFlow("""
 362 // First, build the bits on RHEL7.2
 363 b = build(params, '${fullBuildJobName}')
 364
 365 // Then, run the perf tests
 366 parallel(
 367     ${flowJobPerfRunList.join(",\n    ")}
 368 )
 369 """)
 370     }
 371
 372     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 373     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 374
 375     if (isPR) {
 376         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 377         builder.setGithubContext("Linux Perf Test Flow")
 378         builder.triggerOnlyOnComment()
 379         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 380         builder.triggerForBranch(branch)
 381         builder.emitTrigger(newFlowJob)
 382     }
 383     else {
 384         // Set a push trigger
 385         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 386         builder.emitTrigger(newFlowJob)
 387     }
 388
 389 } // isPR
 390
 391 def static getFullThroughputJobName(def project, def os, def isPR) {
 392     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 393 }
 394
 395 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 396 [true, false].each { isPR ->
 397     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 398     def architecture = 'x64'
 399     def configuration = 'Release'
 400
 401     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 402     ['RHEL7.2'].each { os ->
 403         def newBuildJob = job(fullBuildJobName) {
 404             steps {
 405                 shell("./build.sh verbose ${architecture} ${configuration}")
 406             }
 407         }
 408         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 409         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 410         Utilities.addArchival(newBuildJob, "bin/Product/**")
 411     }
 412
 413     // Actual perf testing on the following OSes
 414     def throughputOSList = ['Ubuntu14.04']
 415     def throughputOptLevelList = ['full_opt', 'min_opt']
 416
 417     def throughputOSOptLevelList = []
 418
 419     throughputOSList.each { os ->
 420         throughputOptLevelList.each { opt_level ->
 421             throughputOSOptLevelList.add("${os}_${opt_level}")
 422         }
 423     }
 424
 425     throughputOSList.each { os ->
 426         throughputOptLevelList.each { opt_level ->
 427             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 428
 429                 label('ubuntu_1604_clr_perf')
 430                     wrappers {
 431                         credentialsBinding {
 432                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 433                         }
 434                     }
 435
 436                 if (isPR) {
 437                     parameters {
 438                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 439                     }
 440                 }
 441
 442                 parameters {
 443                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 444                 }
 445
 446                 def osGroup = getOSGroup(os)
 447                 def runType = isPR ? 'private' : 'rolling'
 448                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 449
 450                 steps {
 451                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 452                     shell("./init-tools.sh")
 453                     copyArtifacts(fullBuildJobName) {
 454                         includePatterns("bin/Product/**")
 455                         buildSelector {
 456                             buildNumber('\${PRODUCT_BUILD}')
 457                         }
 458                     }
 459                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 460                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 461                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 462                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 463                     -arch \"${architecture}\" \\
 464                     -os \"${os}\" \\
 465                     -configuration \"${configuration}\" \\
 466                     -opt_level \"${opt_level}\" \\
 467                     -clr_root \"\${WORKSPACE}\" \\
 468                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 469                     -run_type \"${runType}\" \\
 470                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 471                 }
 472             }
 473
 474             // Save machinedata.json to /artifact/bin/ Jenkins dir
 475             def archiveSettings = new ArchivalSettings()
 476             archiveSettings.addFiles('throughput-*.csv')
 477             archiveSettings.addFiles('machinedata.json')
 478             archiveSettings.setAlwaysArchive()
 479             Utilities.addArchival(newJob, archiveSettings)
 480
 481             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 482
 483             // For perf, we need to keep the run results longer
 484             newJob.with {
 485                 // Enable the log rotator
 486                 logRotator {
 487                     artifactDaysToKeep(7)
 488                     daysToKeep(300)
 489                     artifactNumToKeep(25)
 490                     numToKeep(1000)
 491                 }
 492             }
 493         } // opt_level
 494     } // os
 495
 496     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 497         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 498     }
 499     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 500         if (isPR) {
 501             parameters {
 502                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 503             }
 504         }
 505         buildFlow("""
 506 // First, build the bits on RHEL7.2
 507 b = build(params, '${fullBuildJobName}')
 508
 509 // Then, run the perf tests
 510 parallel(
 511     ${flowJobTPRunList.join(",\n    ")}
 512 )
 513 """)
 514     }
 515
 516     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 517     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 518
 519     if (isPR) {
 520         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 521         builder.setGithubContext("Linux Throughput Perf Test Flow")
 522         builder.triggerOnlyOnComment()
 523         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 524         builder.triggerForBranch(branch)
 525         builder.emitTrigger(newFlowJob)
 526     }
 527     else {
 528         // Set a push trigger
 529         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 530         builder.emitTrigger(newFlowJob)
 531     }
 532
 533 } // isPR
 534
 535 // Setup CoreCLR-Scenarios tests
 536 [true, false].each { isPR ->
 537     ['Windows_NT'].each { os ->
 538         ['x64', 'x86'].each { arch ->
 539             ['ryujit'].each { jit ->
 540                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 541                     def architecture = arch
 542                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 543
 544                         def testEnv = ""
 545
 546                         // Set the label.
 547                         label('windows_server_2016_clr_perf')
 548                         wrappers {
 549                             credentialsBinding {
 550                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 551                             }
 552                         }
 553
 554                         if (isPR) {
 555                             parameters {
 556                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 557                             }
 558                         }
 559
 560                         parameters {
 561                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 562                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 563                         }
 564
 565                         def configuration = 'Release'
 566                         def runType = isPR ? 'private' : 'rolling'
 567                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 568                         def uploadString = '-uploadToBenchview'
 569
 570                         steps {
 571                             // Batch
 572                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 573                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 574                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 575
 576                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 577                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 578                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 579                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 580                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 581                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 582                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 583                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 584                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 585
 586                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 587
 588                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
 589
 590                             // Profile=Off
 591                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 592
 593                             // Profile=On
 594                             if (opt_level != 'min_opt') {
 595                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 596                             }
 597                         }
 598                     }
 599
 600                     def archiveSettings = new ArchivalSettings()
 601                     archiveSettings.addFiles('bin/sandbox_logs/**')
 602                     archiveSettings.addFiles('machinedata.json')
 603                     archiveSettings.setAlwaysArchive()
 604
 605                     Utilities.addArchival(newJob, archiveSettings)
 606                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 607
 608                     newJob.with {
 609                         logRotator {
 610                             artifactDaysToKeep(30)
 611                             daysToKeep(30)
 612                             artifactNumToKeep(200)
 613                             numToKeep(200)
 614                         }
 615                         wrappers {
 616                             timeout {
 617                                 absolute(240)
 618                             }
 619                         }
 620                     }
 621
 622                     if (isPR) {
 623                         def opts = ""
 624                         if (opt_level == 'min_opt') {
 625                             opts = '\\W+min_opts'
 626                         }
 627                         def jitt = ""
 628                         if (jit != 'ryujit') {
 629                             jitt = "\\W+${jit}"
 630                         }
 631
 632                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 633                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 634                         builder.triggerOnlyOnComment()
 635                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 636                         builder.triggerForBranch(branch)
 637                         builder.emitTrigger(newJob)
 638                     }
 639                     else if (opt_level == 'full_opt') {
 640                         // Set a push trigger
 641                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 642                         builder.emitTrigger(newJob)
 643                     }
 644                     else {
 645                         // Set periodic trigger
 646                         Utilities.addPeriodicTrigger(newJob, '@daily')
 647                     }
 648                 }
 649             }
 650         }
 651     }
 652 }
 653
 654 // Setup size-on-disk test
 655 ['Windows_NT'].each { os ->
 656     ['x64', 'x86'].each { arch ->
 657         def architecture = arch
 658         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 659
 660             wrappers {
 661                 credentialsBinding {
 662                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 663                 }
 664             }
 665
 666             def channel = 'master'
 667             def configuration = 'Release'
 668             def runType = 'rolling'
 669             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 670             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 671             def coreRoot = "${testBin}\\Tests\\Core_Root"
 672             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 673
 674             steps {
 675                 // Install nuget and get BenchView tools
 676                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 677                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 678                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 679
 680                 // Generate submission metadata for BenchView
 681                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 682                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 683                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 684                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 685                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 686                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 687                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 688
 689                 // Generate machine data from BenchView
 690                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 691
 692                 // Build CoreCLR and gnerate test layout
 693                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 694                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 695
 696                 // Run the size on disk benchmark
 697                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 698
 699                 // From sodbench.csv, create measurment.json, then submission.json
 700                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 701                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 702
 703                 // If this is a PR, upload submission.json
 704                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 705             }
 706         }
 707
 708         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 709
 710         def archiveSettings = new ArchivalSettings()
 711         archiveSettings.addFiles('bin/toArchive/**')
 712         archiveSettings.addFiles('machinedata.json')
 713         archiveSettings.setAlwaysArchive()
 714
 715         Utilities.addArchival(newJob, archiveSettings)
 716         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 717
 718         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 719         //Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 720
 721         newJob.with {
 722             logRotator {
 723                 artifactDaysToKeep(30)
 724                 daysToKeep(30)
 725                 artifactNumToKeep(200)
 726                 numToKeep(200)
 727             }
 728             wrappers {
 729                 timeout {
 730                     absolute(240)
 731                 }
 732             }
 733         }
 734     }
 735 }
 736
 737 // Setup IlLink tests
 738 [true, false].each { isPR ->
 739     ['Windows_NT'].each { os ->
 740         ['x64'].each { arch ->
 741             ['ryujit'].each { jit ->
 742                 ['full_opt'].each { opt_level ->
 743                     def architecture = arch
 744                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 745
 746                         def testEnv = ""
 747                         wrappers {
 748                             credentialsBinding {
 749                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 750                             }
 751                         }
 752
 753                         if (isPR) {
 754                             parameters {
 755                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 756                             }
 757                         }
 758
 759                         parameters {
 760                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 761                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 762                         }
 763
 764                         def configuration = 'Release'
 765                         def runType = isPR ? 'private' : 'rolling'
 766                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 767                         def uploadString = '-uploadToBenchview'
 768
 769                         steps {
 770                             // Batch
 771                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 772                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 773                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 774
 775                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 776                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 777                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 778                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 779                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 780                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 781                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 782                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 783                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 784
 785                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 786
 787                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 788
 789                             // Scenario: ILLink
 790                             batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
 791                             "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 792                         }
 793                     }
 794
 795                     def archiveSettings = new ArchivalSettings()
 796                     archiveSettings.addFiles('bin/sandbox_logs/**')
 797                     archiveSettings.addFiles('machinedata.json')
 798                     archiveSettings.setAlwaysArchive()
 799
 800                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 801                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 802                     Utilities.addArchival(newJob, archiveSettings)
 803                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 804
 805                     newJob.with {
 806                         logRotator {
 807                             artifactDaysToKeep(30)
 808                             daysToKeep(30)
 809                             artifactNumToKeep(200)
 810                             numToKeep(200)
 811                         }
 812                         wrappers {
 813                             timeout {
 814                                 absolute(240)
 815                             }
 816                         }
 817                     }
 818
 819                     if (isPR) {
 820                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 821                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 822                         builder.triggerOnlyOnComment()
 823                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 824                         builder.triggerForBranch(branch)
 825                         builder.emitTrigger(newJob)
 826                     }
 827                     else {
 828                         // Set a push trigger
 829                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 830                         builder.emitTrigger(newJob)
 831                     }
 832                 }
 833             }
 834         }
 835     }
 836 }
 837
 838 Utilities.createHelperJob(this, project, branch,
 839     "Welcome to the ${project} Perf help",
 840     "Have a nice day!")