perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             ['ryujit'].each { jit ->
  33                 if (arch == 'x64' && jit == 'legacy_backend') {
  34                     return
  35                 }
  36
  37                 ['full_opt', 'min_opt'].each { opt_level ->
  38
  39                     def architecture = arch
  40                     def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  41                     def testEnv = ""
  42
  43                     if (jit == 'legacy_backend') {
  44                         testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
  45                     }
  46
  47                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  48                         // Set the label.
  49                         label('windows_server_2016_clr_perf')
  50                         wrappers {
  51                             credentialsBinding {
  52                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  53                             }
  54                         }
  55
  56                         if (isPR) {
  57                             parameters {
  58                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  59                             }
  60                         }
  61
  62                         parameters {
  63                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  64                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  65                         }
  66
  67                         def configuration = 'Release'
  68                         def runType = isPR ? 'private' : 'rolling'
  69                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  70                         def uploadString = '-uploadToBenchview'
  71
  72                         steps {
  73                             // Batch
  74
  75                             batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  76                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  77                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  78                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  79                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  80                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  81                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  82                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  83                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  84                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  85                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  86                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  87
  88                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  89
  90                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
  91
  92                             // Run with just stopwatch: Profile=Off
  93                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  94                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  95
  96                             // Run with the full set of counters enabled: Profile=On
  97                             if (opt_level != 'min_opt') {
  98                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  99                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 100                             }
 101                         }
 102                     }
 103
 104                     def archiveSettings = new ArchivalSettings()
 105                     archiveSettings.addFiles('bin/sandbox_logs/**')
 106                     archiveSettings.addFiles('machinedata.json')
 107
 108                     Utilities.addArchival(newJob, archiveSettings)
 109                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 110
 111                     newJob.with {
 112                         logRotator {
 113                             artifactDaysToKeep(30)
 114                             daysToKeep(30)
 115                             artifactNumToKeep(200)
 116                             numToKeep(200)
 117                         }
 118                         wrappers {
 119                             timeout {
 120                                 absolute(240)
 121                             }
 122                         }
 123                     }
 124
 125                     if (isPR) {
 126                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 127                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 128
 129                         def opts = ""
 130                         if (opt_level == 'min_opt') {
 131                             opts = '\\W+min_opts'
 132                         }
 133                         def jitt = ""
 134                         if (jit != 'ryujit') {
 135                             jitt = "\\W+${jit}"
 136                         }
 137
 138                         builder.triggerOnlyOnComment()
 139                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 140                         builder.triggerForBranch(branch)
 141                         builder.emitTrigger(newJob)
 142                     }
 143                     else {
 144                         // Set a push trigger
 145                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 146                         builder.emitTrigger(newJob)
 147                     }
 148                 }
 149             }
 150         }
 151     }
 152 }
 153
 154 // Setup throughput perflab tests runs
 155 [true, false].each { isPR ->
 156     ['Windows_NT'].each { os ->
 157         ['x64', 'x86'].each { arch ->
 158             ['ryujit', 'legacy_backend'].each { jit ->
 159                 [true, false].each { pgo_optimized ->
 160                     if (arch == 'x64' && jit == 'legacy_backend') {
 161                         return
 162                     }
 163
 164                     // pgo not supported for legacy_backend
 165                     if (pgo_optimized && jit == 'legacy_backend') {
 166                         return
 167                     }
 168
 169                     ['full_opt', 'min_opt'].each { opt_level ->
 170                         def architecture = arch
 171
 172                         pgo_build = ""
 173                         pgo_test = ""
 174                         pgo_string = "pgo"
 175                         if (!pgo_optimized) {
 176                             pgo_build = " -nopgooptimize"
 177                             pgo_test = " -nopgo"
 178                             pgo_string = "nopgo"
 179                         }
 180
 181                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 182                             // Set the label.
 183                             label('windows_server_2016_clr_perf')
 184                             wrappers {
 185                                 credentialsBinding {
 186                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 187                                 }
 188                             }
 189
 190                             if (isPR) {
 191                                 parameters {
 192                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 193                                 }
 194                             }
 195
 196                             def configuration = 'Release'
 197                             def runType = isPR ? 'private' : 'rolling'
 198                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 199
 200                             steps {
 201                                 // Batch
 202                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 203                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 204                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 205                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 206                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 207                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 208                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 209                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 210                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 211                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 212                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 213                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 214                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 215                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 216                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 217                             }
 218                         }
 219
 220                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 221                         def archiveSettings = new ArchivalSettings()
 222                         archiveSettings.addFiles('throughput-*.csv')
 223                         Utilities.addArchival(newJob, archiveSettings)
 224
 225                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 226
 227                         if (isPR) {
 228                             def opts = ""
 229                             if (opt_level == 'min_opt') {
 230                                 opts = '\\W+min_opts'
 231                             }
 232
 233                             def jitt = ""
 234                             if (jit != 'ryujit') {
 235                                 jitt = "\\W+${jit}"
 236                             }
 237
 238                             def pgo_trigger = ""
 239                             if (pgo_optimized) {
 240                                 pgo_trigger = "\\W+nopgo"
 241                             }
 242
 243
 244                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 245                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 246                             builder.triggerOnlyOnComment()
 247                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 248                             builder.triggerForBranch(branch)
 249                             builder.emitTrigger(newJob)
 250                         }
 251                         else {
 252                             // Set a push trigger
 253                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 254                             builder.emitTrigger(newJob)
 255                         }
 256                     }
 257                 }
 258             }
 259         }
 260     }
 261 }
 262
 263 def static getFullPerfJobName(def project, def os, def isPR) {
 264     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 265 }
 266
 267 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 268 [true, false].each { isPR ->
 269     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 270     def architecture = 'x64'
 271     def configuration = 'Release'
 272
 273     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 274     ['RHEL7.2'].each { os ->
 275         def newBuildJob = job(fullBuildJobName) {
 276             steps {
 277                 shell("./build.sh verbose ${architecture} ${configuration}")
 278             }
 279         }
 280         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 281         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 282         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 283     }
 284
 285
 286     // Actual perf testing on the following OSes
 287     def perfOSList = ['Ubuntu14.04']
 288     perfOSList.each { os ->
 289         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 290
 291             label('ubuntu_1604_clr_perf')
 292             wrappers {
 293                 credentialsBinding {
 294                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 295                 }
 296             }
 297
 298             if (isPR) {
 299                 parameters {
 300                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 301                 }
 302             }
 303
 304             parameters {
 305                 // Cap the maximum number of iterations to 21.
 306                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 307                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 308                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 309             }
 310
 311             def osGroup = getOSGroup(os)
 312             def runType = isPR ? 'private' : 'rolling'
 313             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 314
 315             steps {
 316                 shell("./tests/scripts/perf-prep.sh")
 317                 shell("./init-tools.sh")
 318                 copyArtifacts(fullBuildJobName) {
 319                     includePatterns("bin/**")
 320                     buildSelector {
 321                         buildNumber('\${PRODUCT_BUILD}')
 322                     }
 323                 }
 324                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 325                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 326                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 327                 shell("""./tests/scripts/run-xunit-perf.sh \\
 328                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 329                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 330                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 331                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 332                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 333                 --runType=\"${runType}\" \\
 334                 --benchViewOS=\"${os}\" \\
 335                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 336                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 337                 --uploadToBenchview""")
 338                 shell("mkdir -p bin/toArchive/sandbox/Logs/")
 339                 shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
 340             }
 341         }
 342
 343         def archiveSettings = new ArchivalSettings()
 344         archiveSettings.addFiles('bin/toArchive/**')
 345         archiveSettings.addFiles('machinedata.json')
 346
 347         Utilities.addArchival(newJob, archiveSettings)
 348         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 349
 350         // For perf, we need to keep the run results longer
 351         newJob.with {
 352             // Enable the log rotator
 353             logRotator {
 354                 artifactDaysToKeep(30)
 355                 daysToKeep(30)
 356                 artifactNumToKeep(200)
 357                 numToKeep(200)
 358             }
 359             wrappers {
 360                 timeout {
 361                     absolute(240)
 362                 }
 363             }
 364         }
 365     } // os
 366
 367     def flowJobPerfRunList = perfOSList.collect { os ->
 368         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 369     }
 370     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 371         if (isPR) {
 372             parameters {
 373                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 374             }
 375         }
 376         buildFlow("""
 377 // First, build the bits on RHEL7.2
 378 b = build(params, '${fullBuildJobName}')
 379
 380 // Then, run the perf tests
 381 parallel(
 382     ${flowJobPerfRunList.join(",\n    ")}
 383 )
 384 """)
 385     }
 386
 387     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 388     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 389
 390     if (isPR) {
 391         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 392         builder.setGithubContext("Linux Perf Test Flow")
 393         builder.triggerOnlyOnComment()
 394         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 395         builder.triggerForBranch(branch)
 396         builder.emitTrigger(newFlowJob)
 397     }
 398     else {
 399         // Set a push trigger
 400         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 401         builder.emitTrigger(newFlowJob)
 402     }
 403
 404 } // isPR
 405
 406 def static getFullThroughputJobName(def project, def os, def isPR) {
 407     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 408 }
 409
 410 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 411 [true, false].each { isPR ->
 412     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 413     def architecture = 'x64'
 414     def configuration = 'Release'
 415
 416     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 417     ['RHEL7.2'].each { os ->
 418         def newBuildJob = job(fullBuildJobName) {
 419             steps {
 420                 shell("./build.sh verbose ${architecture} ${configuration}")
 421             }
 422         }
 423         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 424         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 425         Utilities.addArchival(newBuildJob, "bin/Product/**")
 426     }
 427
 428     // Actual perf testing on the following OSes
 429     def throughputOSList = ['Ubuntu14.04']
 430     def throughputOptLevelList = ['full_opt', 'min_opt']
 431
 432     def throughputOSOptLevelList = []
 433
 434     throughputOSList.each { os ->
 435         throughputOptLevelList.each { opt_level ->
 436             throughputOSOptLevelList.add("${os}_${opt_level}")
 437         }
 438     }
 439
 440     throughputOSList.each { os ->
 441         throughputOptLevelList.each { opt_level ->
 442             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 443
 444                 label('ubuntu_1604_clr_perf')
 445                     wrappers {
 446                         credentialsBinding {
 447                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 448                         }
 449                     }
 450
 451                 if (isPR) {
 452                     parameters {
 453                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 454                     }
 455                 }
 456
 457                 parameters {
 458                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 459                 }
 460
 461                 def osGroup = getOSGroup(os)
 462                 def runType = isPR ? 'private' : 'rolling'
 463                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 464
 465                 steps {
 466                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 467                     shell("./init-tools.sh")
 468                     copyArtifacts(fullBuildJobName) {
 469                         includePatterns("bin/Product/**")
 470                         buildSelector {
 471                             buildNumber('\${PRODUCT_BUILD}')
 472                         }
 473                     }
 474                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 475                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 476                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 477                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 478                     -arch \"${architecture}\" \\
 479                     -os \"${os}\" \\
 480                     -configuration \"${configuration}\" \\
 481                     -opt_level \"${opt_level}\" \\
 482                     -clr_root \"\${WORKSPACE}\" \\
 483                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 484                     -run_type \"${runType}\" \\
 485                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 486                 }
 487             }
 488
 489             // Save machinedata.json to /artifact/bin/ Jenkins dir
 490             def archiveSettings = new ArchivalSettings()
 491             archiveSettings.addFiles('throughput-*.csv')
 492             archiveSettings.addFiles('machinedata.json')
 493             Utilities.addArchival(newJob, archiveSettings)
 494
 495             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 496
 497             // For perf, we need to keep the run results longer
 498             newJob.with {
 499                 // Enable the log rotator
 500                 logRotator {
 501                     artifactDaysToKeep(7)
 502                     daysToKeep(300)
 503                     artifactNumToKeep(25)
 504                     numToKeep(1000)
 505                 }
 506             }
 507         } // opt_level
 508     } // os
 509
 510     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 511         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 512     }
 513     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 514         if (isPR) {
 515             parameters {
 516                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 517             }
 518         }
 519         buildFlow("""
 520 // First, build the bits on RHEL7.2
 521 b = build(params, '${fullBuildJobName}')
 522
 523 // Then, run the perf tests
 524 parallel(
 525     ${flowJobTPRunList.join(",\n    ")}
 526 )
 527 """)
 528     }
 529
 530     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 531     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 532
 533     if (isPR) {
 534         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 535         builder.setGithubContext("Linux Throughput Perf Test Flow")
 536         builder.triggerOnlyOnComment()
 537         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 538         builder.triggerForBranch(branch)
 539         builder.emitTrigger(newFlowJob)
 540     }
 541     else {
 542         // Set a push trigger
 543         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 544         builder.emitTrigger(newFlowJob)
 545     }
 546
 547 } // isPR
 548
 549 // Setup CoreCLR-Scenarios tests
 550 [true, false].each { isPR ->
 551     ['Windows_NT'].each { os ->
 552         ['x64', 'x86'].each { arch ->
 553             ['ryujit'].each { jit ->
 554                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 555                     def architecture = arch
 556                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 557
 558                         def testEnv = ""
 559
 560                         // Set the label.
 561                         label('windows_server_2016_clr_perf')
 562                         wrappers {
 563                             credentialsBinding {
 564                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 565                             }
 566                         }
 567
 568                         if (isPR) {
 569                             parameters {
 570                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 571                             }
 572                         }
 573
 574                         parameters {
 575                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 576                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 577                         }
 578
 579                         def configuration = 'Release'
 580                         def runType = isPR ? 'private' : 'rolling'
 581                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 582                         def uploadString = '-uploadToBenchview'
 583
 584                         steps {
 585                             // Batch
 586                             batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 587                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 588                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 589
 590                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 591                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 592                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 593                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 594                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 595                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 596                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 597                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 598                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 599
 600                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 601
 602                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
 603
 604                             // Profile=Off
 605                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 606
 607                             // Profile=On
 608                             if (opt_level != 'min_opt') {
 609                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 610                             }
 611                         }
 612                     }
 613
 614                     def archiveSettings = new ArchivalSettings()
 615                     archiveSettings.addFiles('bin/sandbox_logs/**')
 616                     archiveSettings.addFiles('machinedata.json')
 617
 618                     Utilities.addArchival(newJob, archiveSettings)
 619                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 620
 621                     newJob.with {
 622                         logRotator {
 623                             artifactDaysToKeep(30)
 624                             daysToKeep(30)
 625                             artifactNumToKeep(200)
 626                             numToKeep(200)
 627                         }
 628                         wrappers {
 629                             timeout {
 630                                 absolute(240)
 631                             }
 632                         }
 633                     }
 634
 635                     if (isPR) {
 636                         def opts = ""
 637                         if (opt_level == 'min_opt') {
 638                             opts = '\\W+min_opts'
 639                         }
 640                         def jitt = ""
 641                         if (jit != 'ryujit') {
 642                             jitt = "\\W+${jit}"
 643                         }
 644
 645                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 646                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 647                         builder.triggerOnlyOnComment()
 648                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 649                         builder.triggerForBranch(branch)
 650                         builder.emitTrigger(newJob)
 651                     }
 652                     else {
 653                         // Set a push trigger
 654                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 655                         builder.emitTrigger(newJob)
 656                     }
 657                 }
 658             }
 659         }
 660     }
 661 }
 662
 663 // Setup size-on-disk test
 664 ['Windows_NT'].each { os ->
 665     ['x64', 'x86'].each { arch ->
 666         def architecture = arch
 667         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 668
 669             wrappers {
 670                 credentialsBinding {
 671                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 672                 }
 673             }
 674
 675             def channel = 'master'
 676             def configuration = 'Release'
 677             def runType = 'rolling'
 678             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 679             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 680             def coreRoot = "${testBin}\\Tests\\Core_Root"
 681             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 682
 683             steps {
 684                 // Install nuget and get BenchView tools
 685                 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 686                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 687                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 688
 689                 // Generate submission metadata for BenchView
 690                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 691                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 692                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 693                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 694                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 695                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 696                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 697
 698                 // Generate machine data from BenchView
 699                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 700
 701                 // Build CoreCLR and gnerate test layout
 702                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 703                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 704
 705                 // Run the size on disk benchmark
 706                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 707
 708                 // From sodbench.csv, create measurment.json, then submission.json
 709                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 710                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 711
 712                 // If this is a PR, upload submission.json
 713                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 714             }
 715         }
 716
 717         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 718
 719         def archiveSettings = new ArchivalSettings()
 720         archiveSettings.addFiles('bin/toArchive/**')
 721         archiveSettings.addFiles('machinedata.json')
 722
 723         Utilities.addArchival(newJob, archiveSettings)
 724         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 725
 726         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 727         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 728
 729         newJob.with {
 730             logRotator {
 731                 artifactDaysToKeep(30)
 732                 daysToKeep(30)
 733                 artifactNumToKeep(200)
 734                 numToKeep(200)
 735             }
 736             wrappers {
 737                 timeout {
 738                     absolute(240)
 739                 }
 740             }
 741         }
 742     }
 743 }
 744
 745 // Setup IlLink tests
 746 [true, false].each { isPR ->
 747     ['Windows_NT'].each { os ->
 748         ['x64'].each { arch ->
 749             ['ryujit'].each { jit ->
 750                 ['full_opt'].each { opt_level ->
 751                     def architecture = arch
 752                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 753
 754                         def testEnv = ""
 755                         wrappers {
 756                             credentialsBinding {
 757                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 758                             }
 759                         }
 760
 761                         if (isPR) {
 762                             parameters {
 763                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 764                             }
 765                         }
 766
 767                         parameters {
 768                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 769                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 770                         }
 771
 772                         def configuration = 'Release'
 773                         def runType = isPR ? 'private' : 'rolling'
 774                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 775                         def uploadString = '-uploadToBenchview'
 776
 777                         steps {
 778                             // Batch
 779                             batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 780                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 781                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 782
 783                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 784                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 785                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 786                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 787                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 788                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 789                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 790                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 791                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 792
 793                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 794
 795                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 796
 797                             // Scenario: ILLink
 798                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 799                         }
 800                     }
 801
 802                     def archiveSettings = new ArchivalSettings()
 803                     archiveSettings.addFiles('bin/sandbox_logs/**')
 804                     archiveSettings.addFiles('machinedata.json')
 805
 806                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 807                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 808                     Utilities.addArchival(newJob, archiveSettings)
 809                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 810
 811                     newJob.with {
 812                         logRotator {
 813                             artifactDaysToKeep(30)
 814                             daysToKeep(30)
 815                             artifactNumToKeep(200)
 816                             numToKeep(200)
 817                         }
 818                         wrappers {
 819                             timeout {
 820                                 absolute(240)
 821                             }
 822                         }
 823                     }
 824
 825                     if (isPR) {
 826                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 827                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 828                         builder.triggerOnlyOnComment()
 829                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 830                         builder.triggerForBranch(branch)
 831                         builder.emitTrigger(newJob)
 832                     }
 833                     else {
 834                         // Set a push trigger
 835                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 836                         builder.emitTrigger(newJob)
 837                     }
 838                 }
 839             }
 840         }
 841     }
 842 }
 843
 844 Utilities.createHelperJob(this, project, branch,
 845     "Welcome to the ${project} Perf help",
 846     "Have a nice day!")