perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             ['ryujit'].each { jit ->
  33                 ['full_opt', 'min_opt'].each { opt_level ->
  34
  35                     def architecture = arch
  36                     def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  37                     def testEnv = ""
  38
  39                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  40                         // Set the label.
  41                         label('windows_server_2016_clr_perf')
  42                         wrappers {
  43                             credentialsBinding {
  44                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  45                             }
  46                         }
  47
  48                         if (isPR) {
  49                             parameters {
  50                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  51                             }
  52                         }
  53
  54                         parameters {
  55                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  56                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  57                         }
  58
  59                         def configuration = 'Release'
  60                         def runType = isPR ? 'private' : 'rolling'
  61                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  62                         def uploadString = '-uploadToBenchview'
  63
  64                         steps {
  65                             // Batch
  66
  67                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  68                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  69                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  70                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  71                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  72                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  73                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  74                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  75                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  76                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  77                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  78                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  79
  80                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  81
  82                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
  83
  84                             // Run with just stopwatch: Profile=Off
  85                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  86                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  87
  88                             // Run with the full set of counters enabled: Profile=On
  89                             if (opt_level != 'min_opt') {
  90                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  91                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  92                             }
  93                         }
  94                     }
  95
  96                     def archiveSettings = new ArchivalSettings()
  97                     archiveSettings.addFiles('bin/sandbox_logs/**')
  98                     archiveSettings.addFiles('machinedata.json')
  99                     archiveSettings.setAlwaysArchive()
 100
 101                     Utilities.addArchival(newJob, archiveSettings)
 102                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 103
 104                     newJob.with {
 105                         logRotator {
 106                             artifactDaysToKeep(30)
 107                             daysToKeep(30)
 108                             artifactNumToKeep(200)
 109                             numToKeep(200)
 110                         }
 111                         wrappers {
 112                             timeout {
 113                                 absolute(240)
 114                             }
 115                         }
 116                     }
 117
 118                     if (isPR) {
 119                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 120                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 121
 122                         def opts = ""
 123                         if (opt_level == 'min_opt') {
 124                             opts = '\\W+min_opts'
 125                         }
 126                         def jitt = ""
 127                         if (jit != 'ryujit') {
 128                             jitt = "\\W+${jit}"
 129                         }
 130
 131                         builder.triggerOnlyOnComment()
 132                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 133                         builder.triggerForBranch(branch)
 134                         builder.emitTrigger(newJob)
 135                     }
 136                     else if (opt_level == 'full_opt') {
 137                         // Set a push trigger
 138                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 139                         builder.emitTrigger(newJob)
 140                     }
 141                     else {
 142                         // Set periodic trigger
 143                         Utilities.addPeriodicTrigger(newJob, '@daily')
 144                     }
 145                 }
 146             }
 147         }
 148     }
 149 }
 150
 151 // Setup throughput perflab tests runs
 152 [true, false].each { isPR ->
 153     ['Windows_NT'].each { os ->
 154         ['x64', 'x86'].each { arch ->
 155             ['ryujit'].each { jit ->
 156                 [true, false].each { pgo_optimized ->
 157                     ['full_opt', 'min_opt'].each { opt_level ->
 158                         def architecture = arch
 159
 160                         pgo_build = ""
 161                         pgo_test = ""
 162                         pgo_string = "pgo"
 163                         if (!pgo_optimized) {
 164                             pgo_build = " -nopgooptimize"
 165                             pgo_test = " -nopgo"
 166                             pgo_string = "nopgo"
 167                         }
 168
 169                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 170                             // Set the label.
 171                             label('windows_server_2016_clr_perf')
 172                             wrappers {
 173                                 credentialsBinding {
 174                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 175                                 }
 176                             }
 177
 178                             if (isPR) {
 179                                 parameters {
 180                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 181                                 }
 182                             }
 183
 184                             def configuration = 'Release'
 185                             def runType = isPR ? 'private' : 'rolling'
 186                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 187
 188                             steps {
 189                                 // Batch
 190                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 191                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 192                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 193                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 194                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 195                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 196                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 197                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 198                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 199                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 200                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 201                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 202                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 203                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 204                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 205                             }
 206                         }
 207
 208                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 209                         def archiveSettings = new ArchivalSettings()
 210                         archiveSettings.addFiles('throughput-*.csv')
 211                         archiveSettings.setAlwaysArchive()
 212                         Utilities.addArchival(newJob, archiveSettings)
 213
 214                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 215
 216                         if (isPR) {
 217                             def opts = ""
 218                             if (opt_level == 'min_opt') {
 219                                 opts = '\\W+min_opts'
 220                             }
 221
 222                             def jitt = ""
 223                             if (jit != 'ryujit') {
 224                                 jitt = "\\W+${jit}"
 225                             }
 226
 227                             def pgo_trigger = ""
 228                             if (pgo_optimized) {
 229                                 pgo_trigger = "\\W+nopgo"
 230                             }
 231
 232
 233                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 234                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 235                             builder.triggerOnlyOnComment()
 236                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 237                             builder.triggerForBranch(branch)
 238                             builder.emitTrigger(newJob)
 239                         }
 240                         else if (opt_level == 'full_opt' && pgo_optimized) {
 241                             // Set a push trigger
 242                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 243                             builder.emitTrigger(newJob)
 244                         }
 245                         else {
 246                             // Set periodic trigger
 247                             Utilities.addPeriodicTrigger(newJob, '@daily')
 248                         }
 249                     }
 250                 }
 251             }
 252         }
 253     }
 254 }
 255
 256 def static getFullPerfJobName(def project, def os, def isPR) {
 257     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 258 }
 259
 260 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 261 [true, false].each { isPR ->
 262     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 263     def architecture = 'x64'
 264     def configuration = 'Release'
 265
 266     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 267     ['RHEL7.2'].each { os ->
 268         def newBuildJob = job(fullBuildJobName) {
 269             steps {
 270                 shell("./build.sh verbose ${architecture} ${configuration}")
 271                 shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}")
 272             }
 273         }
 274         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 275         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 276         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
 277     }
 278
 279
 280     // Actual perf testing on the following OSes
 281     def perfOSList = ['Ubuntu16.04']
 282     perfOSList.each { os ->
 283         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 284
 285             label('ubuntu_1604_clr_perf')
 286             wrappers {
 287                 credentialsBinding {
 288                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 289                 }
 290             }
 291
 292             if (isPR) {
 293                 parameters {
 294                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 295                 }
 296             }
 297
 298             parameters {
 299                 // Cap the maximum number of iterations to 21.
 300                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 301                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 302                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 303             }
 304
 305             def osGroup = getOSGroup(os)
 306             def runType = isPR ? 'private' : 'rolling'
 307             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 308             def uploadString = '-uploadToBenchview'
 309
 310             def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
 311
 312             steps {
 313                 shell("./tests/scripts/perf-prep.sh --nocorefx")
 314                 shell("./init-tools.sh")
 315                 copyArtifacts(fullBuildJobName) {
 316                     includePatterns("bin/**")
 317                     buildSelector {
 318                         buildNumber('\${PRODUCT_BUILD}')
 319                     }
 320                 }
 321                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 322                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 323                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 324                 shell("""python3 ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
 325             }
 326         }
 327
 328         def archiveSettings = new ArchivalSettings()
 329         archiveSettings.addFiles('bin/sandbox_logs/**')
 330         archiveSettings.addFiles('machinedata.json')
 331         archiveSettings.setAlwaysArchive()
 332
 333         Utilities.addArchival(newJob, archiveSettings)
 334         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 335
 336         // For perf, we need to keep the run results longer
 337         newJob.with {
 338             // Enable the log rotator
 339             logRotator {
 340                 artifactDaysToKeep(30)
 341                 daysToKeep(30)
 342                 artifactNumToKeep(200)
 343                 numToKeep(200)
 344             }
 345             wrappers {
 346                 timeout {
 347                     absolute(240)
 348                 }
 349             }
 350         }
 351     } // os
 352
 353     def flowJobPerfRunList = perfOSList.collect { os ->
 354         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 355     }
 356     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 357         if (isPR) {
 358             parameters {
 359                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 360             }
 361         }
 362         buildFlow("""
 363 // First, build the bits on RHEL7.2
 364 b = build(params, '${fullBuildJobName}')
 365
 366 // Then, run the perf tests
 367 parallel(
 368     ${flowJobPerfRunList.join(",\n    ")}
 369 )
 370 """)
 371     }
 372
 373     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 374     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 375
 376     if (isPR) {
 377         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 378         builder.setGithubContext("Linux Perf Test Flow")
 379         builder.triggerOnlyOnComment()
 380         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 381         builder.triggerForBranch(branch)
 382         builder.emitTrigger(newFlowJob)
 383     }
 384     else {
 385         // Set a push trigger
 386         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 387         builder.emitTrigger(newFlowJob)
 388     }
 389
 390 } // isPR
 391
 392 def static getFullThroughputJobName(def project, def os, def isPR) {
 393     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 394 }
 395
 396 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 397 [true, false].each { isPR ->
 398     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 399     def architecture = 'x64'
 400     def configuration = 'Release'
 401
 402     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 403     ['RHEL7.2'].each { os ->
 404         def newBuildJob = job(fullBuildJobName) {
 405             steps {
 406                 shell("./build.sh verbose ${architecture} ${configuration}")
 407             }
 408         }
 409         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 410         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 411         Utilities.addArchival(newBuildJob, "bin/Product/**")
 412     }
 413
 414     // Actual perf testing on the following OSes
 415     def throughputOSList = ['Ubuntu14.04']
 416     def throughputOptLevelList = ['full_opt', 'min_opt']
 417
 418     def throughputOSOptLevelList = []
 419
 420     throughputOSList.each { os ->
 421         throughputOptLevelList.each { opt_level ->
 422             throughputOSOptLevelList.add("${os}_${opt_level}")
 423         }
 424     }
 425
 426     throughputOSList.each { os ->
 427         throughputOptLevelList.each { opt_level ->
 428             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 429
 430                 label('ubuntu_1604_clr_perf')
 431                     wrappers {
 432                         credentialsBinding {
 433                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 434                         }
 435                     }
 436
 437                 if (isPR) {
 438                     parameters {
 439                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 440                     }
 441                 }
 442
 443                 parameters {
 444                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 445                 }
 446
 447                 def osGroup = getOSGroup(os)
 448                 def runType = isPR ? 'private' : 'rolling'
 449                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 450
 451                 steps {
 452                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 453                     shell("./init-tools.sh")
 454                     copyArtifacts(fullBuildJobName) {
 455                         includePatterns("bin/Product/**")
 456                         buildSelector {
 457                             buildNumber('\${PRODUCT_BUILD}')
 458                         }
 459                     }
 460                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 461                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 462                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 463                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 464                     -arch \"${architecture}\" \\
 465                     -os \"${os}\" \\
 466                     -configuration \"${configuration}\" \\
 467                     -opt_level \"${opt_level}\" \\
 468                     -clr_root \"\${WORKSPACE}\" \\
 469                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 470                     -run_type \"${runType}\" \\
 471                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 472                 }
 473             }
 474
 475             // Save machinedata.json to /artifact/bin/ Jenkins dir
 476             def archiveSettings = new ArchivalSettings()
 477             archiveSettings.addFiles('throughput-*.csv')
 478             archiveSettings.addFiles('machinedata.json')
 479             archiveSettings.setAlwaysArchive()
 480             Utilities.addArchival(newJob, archiveSettings)
 481
 482             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 483
 484             // For perf, we need to keep the run results longer
 485             newJob.with {
 486                 // Enable the log rotator
 487                 logRotator {
 488                     artifactDaysToKeep(7)
 489                     daysToKeep(300)
 490                     artifactNumToKeep(25)
 491                     numToKeep(1000)
 492                 }
 493             }
 494         } // opt_level
 495     } // os
 496
 497     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 498         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 499     }
 500     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 501         if (isPR) {
 502             parameters {
 503                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 504             }
 505         }
 506         buildFlow("""
 507 // First, build the bits on RHEL7.2
 508 b = build(params, '${fullBuildJobName}')
 509
 510 // Then, run the perf tests
 511 parallel(
 512     ${flowJobTPRunList.join(",\n    ")}
 513 )
 514 """)
 515     }
 516
 517     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 518     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 519
 520     if (isPR) {
 521         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 522         builder.setGithubContext("Linux Throughput Perf Test Flow")
 523         builder.triggerOnlyOnComment()
 524         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 525         builder.triggerForBranch(branch)
 526         builder.emitTrigger(newFlowJob)
 527     }
 528     else {
 529         // Set a push trigger
 530         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 531         builder.emitTrigger(newFlowJob)
 532     }
 533
 534 } // isPR
 535
 536 // Setup CoreCLR-Scenarios tests
 537 [true, false].each { isPR ->
 538     ['Windows_NT'].each { os ->
 539         ['x64', 'x86'].each { arch ->
 540             ['ryujit'].each { jit ->
 541                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 542                     def architecture = arch
 543                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 544
 545                         def testEnv = ""
 546
 547                         // Set the label.
 548                         label('windows_server_2016_clr_perf')
 549                         wrappers {
 550                             credentialsBinding {
 551                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 552                             }
 553                         }
 554
 555                         if (isPR) {
 556                             parameters {
 557                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 558                             }
 559                         }
 560
 561                         parameters {
 562                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 563                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 564                         }
 565
 566                         def configuration = 'Release'
 567                         def runType = isPR ? 'private' : 'rolling'
 568                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 569                         def uploadString = '-uploadToBenchview'
 570
 571                         steps {
 572                             // Batch
 573                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 574                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 575                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 576
 577                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 578                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 579                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 580                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 581                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 582                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 583                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 584                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 585                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 586
 587                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 588
 589                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
 590
 591                             // Profile=Off
 592                             batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 593
 594                             // Profile=On
 595                             if (opt_level != 'min_opt') {
 596                                 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 597                             }
 598                         }
 599                     }
 600
 601                     def archiveSettings = new ArchivalSettings()
 602                     archiveSettings.addFiles('bin/sandbox_logs/**')
 603                     archiveSettings.addFiles('machinedata.json')
 604                     archiveSettings.setAlwaysArchive()
 605
 606                     Utilities.addArchival(newJob, archiveSettings)
 607                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 608
 609                     newJob.with {
 610                         logRotator {
 611                             artifactDaysToKeep(30)
 612                             daysToKeep(30)
 613                             artifactNumToKeep(200)
 614                             numToKeep(200)
 615                         }
 616                         wrappers {
 617                             timeout {
 618                                 absolute(240)
 619                             }
 620                         }
 621                     }
 622
 623                     if (isPR) {
 624                         def opts = ""
 625                         if (opt_level == 'min_opt') {
 626                             opts = '\\W+min_opts'
 627                         }
 628                         def jitt = ""
 629                         if (jit != 'ryujit') {
 630                             jitt = "\\W+${jit}"
 631                         }
 632
 633                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 634                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 635                         builder.triggerOnlyOnComment()
 636                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 637                         builder.triggerForBranch(branch)
 638                         builder.emitTrigger(newJob)
 639                     }
 640                     else if (opt_level == 'full_opt') {
 641                         // Set a push trigger
 642                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 643                         builder.emitTrigger(newJob)
 644                     }
 645                     else {
 646                         // Set periodic trigger
 647                         Utilities.addPeriodicTrigger(newJob, '@daily')
 648                     }
 649                 }
 650             }
 651         }
 652     }
 653 }
 654
 655 // Setup size-on-disk test
 656 ['Windows_NT'].each { os ->
 657     ['x64', 'x86'].each { arch ->
 658         def architecture = arch
 659         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 660
 661             wrappers {
 662                 credentialsBinding {
 663                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 664                 }
 665             }
 666
 667             def channel = 'master'
 668             def configuration = 'Release'
 669             def runType = 'rolling'
 670             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 671             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 672             def coreRoot = "${testBin}\\Tests\\Core_Root"
 673             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 674
 675             steps {
 676                 // Install nuget and get BenchView tools
 677                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 678                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 679                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 680
 681                 // Generate submission metadata for BenchView
 682                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 683                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 684                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 685                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 686                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 687                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 688                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 689
 690                 // Generate machine data from BenchView
 691                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 692
 693                 // Build CoreCLR and gnerate test layout
 694                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 695                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 696
 697                 // Run the size on disk benchmark
 698                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 699
 700                 // From sodbench.csv, create measurment.json, then submission.json
 701                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 702                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 703
 704                 // If this is a PR, upload submission.json
 705                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 706             }
 707         }
 708
 709         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 710
 711         def archiveSettings = new ArchivalSettings()
 712         archiveSettings.addFiles('bin/toArchive/**')
 713         archiveSettings.addFiles('machinedata.json')
 714         archiveSettings.setAlwaysArchive()
 715
 716         Utilities.addArchival(newJob, archiveSettings)
 717         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 718
 719         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 720         //Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 721
 722         newJob.with {
 723             logRotator {
 724                 artifactDaysToKeep(30)
 725                 daysToKeep(30)
 726                 artifactNumToKeep(200)
 727                 numToKeep(200)
 728             }
 729             wrappers {
 730                 timeout {
 731                     absolute(240)
 732                 }
 733             }
 734         }
 735     }
 736 }
 737
 738 // Setup IlLink tests
 739 [true, false].each { isPR ->
 740     ['Windows_NT'].each { os ->
 741         ['x64'].each { arch ->
 742             ['ryujit'].each { jit ->
 743                 ['full_opt'].each { opt_level ->
 744                     def architecture = arch
 745                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 746
 747                         def testEnv = ""
 748                         wrappers {
 749                             credentialsBinding {
 750                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 751                             }
 752                         }
 753
 754                         if (isPR) {
 755                             parameters {
 756                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 757                             }
 758                         }
 759
 760                         parameters {
 761                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 762                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 763                         }
 764
 765                         def configuration = 'Release'
 766                         def runType = isPR ? 'private' : 'rolling'
 767                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 768                         def uploadString = '-uploadToBenchview'
 769
 770                         steps {
 771                             // Batch
 772                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 773                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 774                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 775
 776                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 777                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 778                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 779                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 780                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 781                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 782                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 783                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 784                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 785
 786                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 787
 788                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 789
 790                             // Scenario: ILLink
 791                             batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
 792                             "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 793                         }
 794                     }
 795
 796                     def archiveSettings = new ArchivalSettings()
 797                     archiveSettings.addFiles('bin/sandbox_logs/**')
 798                     archiveSettings.addFiles('machinedata.json')
 799                     archiveSettings.setAlwaysArchive()
 800
 801                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 802                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 803                     Utilities.addArchival(newJob, archiveSettings)
 804                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 805
 806                     newJob.with {
 807                         logRotator {
 808                             artifactDaysToKeep(30)
 809                             daysToKeep(30)
 810                             artifactNumToKeep(200)
 811                             numToKeep(200)
 812                         }
 813                         wrappers {
 814                             timeout {
 815                                 absolute(240)
 816                             }
 817                         }
 818                     }
 819
 820                     if (isPR) {
 821                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 822                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 823                         builder.triggerOnlyOnComment()
 824                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 825                         builder.triggerForBranch(branch)
 826                         builder.emitTrigger(newJob)
 827                     }
 828                     else {
 829                         // Set a push trigger
 830                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 831                         builder.emitTrigger(newJob)
 832                     }
 833                 }
 834             }
 835         }
 836     }
 837 }
 838
 839 Utilities.createHelperJob(this, project, branch,
 840     "Welcome to the ${project} Perf help",
 841     "Have a nice day!")