perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             ['ryujit'].each { jit ->
  33                 ['full_opt', 'min_opt'].each { opt_level ->
  34
  35                     def architecture = arch
  36                     def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
  37                     def testEnv = ""
  38
  39                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  40                         // Set the label.
  41                         label('windows_server_2016_clr_perf')
  42                         wrappers {
  43                             credentialsBinding {
  44                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  45                             }
  46                         }
  47
  48                         if (isPR) {
  49                             parameters {
  50                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  51                             }
  52                         }
  53
  54                         parameters {
  55                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  56                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
  57                         }
  58
  59                         def configuration = 'Release'
  60                         def runType = isPR ? 'private' : 'rolling'
  61                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  62                         def uploadString = '-uploadToBenchview'
  63
  64                         steps {
  65                             // Batch
  66
  67                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  68                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  69                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  70                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  71                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  72                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  73                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  74                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
  75                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
  76                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  77                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  78                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  79
  80                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  81
  82                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
  83
  84                             // Run with just stopwatch: Profile=Off
  85                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  86                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  87
  88                             // Run with the full set of counters enabled: Profile=On
  89                             if (opt_level != 'min_opt') {
  90                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  91                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  92                             }
  93                         }
  94                     }
  95
  96                     def archiveSettings = new ArchivalSettings()
  97                     archiveSettings.addFiles('bin/sandbox_logs/**')
  98                     archiveSettings.addFiles('machinedata.json')
  99
 100                     Utilities.addArchival(newJob, archiveSettings)
 101                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 102
 103                     newJob.with {
 104                         logRotator {
 105                             artifactDaysToKeep(30)
 106                             daysToKeep(30)
 107                             artifactNumToKeep(200)
 108                             numToKeep(200)
 109                         }
 110                         wrappers {
 111                             timeout {
 112                                 absolute(240)
 113                             }
 114                         }
 115                     }
 116
 117                     if (isPR) {
 118                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 119                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 120
 121                         def opts = ""
 122                         if (opt_level == 'min_opt') {
 123                             opts = '\\W+min_opts'
 124                         }
 125                         def jitt = ""
 126                         if (jit != 'ryujit') {
 127                             jitt = "\\W+${jit}"
 128                         }
 129
 130                         builder.triggerOnlyOnComment()
 131                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
 132                         builder.triggerForBranch(branch)
 133                         builder.emitTrigger(newJob)
 134                     }
 135                     else if (opt_level == 'full_opt') {
 136                         // Set a push trigger
 137                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 138                         builder.emitTrigger(newJob)
 139                     }
 140                     else {
 141                         // Set periodic trigger
 142                         Utilities.addPeriodicTrigger(newJob, '@daily')
 143                     }
 144                 }
 145             }
 146         }
 147     }
 148 }
 149
 150 // Setup throughput perflab tests runs
 151 [true, false].each { isPR ->
 152     ['Windows_NT'].each { os ->
 153         ['x64', 'x86'].each { arch ->
 154             ['ryujit'].each { jit ->
 155                 [true, false].each { pgo_optimized ->
 156                     ['full_opt', 'min_opt'].each { opt_level ->
 157                         def architecture = arch
 158
 159                         pgo_build = ""
 160                         pgo_test = ""
 161                         pgo_string = "pgo"
 162                         if (!pgo_optimized) {
 163                             pgo_build = " -nopgooptimize"
 164                             pgo_test = " -nopgo"
 165                             pgo_string = "nopgo"
 166                         }
 167
 168                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
 169                             // Set the label.
 170                             label('windows_server_2016_clr_perf')
 171                             wrappers {
 172                                 credentialsBinding {
 173                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 174                                 }
 175                             }
 176
 177                             if (isPR) {
 178                                 parameters {
 179                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 180                                 }
 181                             }
 182
 183                             def configuration = 'Release'
 184                             def runType = isPR ? 'private' : 'rolling'
 185                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 186
 187                             steps {
 188                                 // Batch
 189                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 190                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 191                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 192                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 193                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 194                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 195                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 196                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 197                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 198                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 199                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 200                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 201                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
 202                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 203                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 204                             }
 205                         }
 206
 207                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 208                         def archiveSettings = new ArchivalSettings()
 209                         archiveSettings.addFiles('throughput-*.csv')
 210                         Utilities.addArchival(newJob, archiveSettings)
 211
 212                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 213
 214                         if (isPR) {
 215                             def opts = ""
 216                             if (opt_level == 'min_opt') {
 217                                 opts = '\\W+min_opts'
 218                             }
 219
 220                             def jitt = ""
 221                             if (jit != 'ryujit') {
 222                                 jitt = "\\W+${jit}"
 223                             }
 224
 225                             def pgo_trigger = ""
 226                             if (pgo_optimized) {
 227                                 pgo_trigger = "\\W+nopgo"
 228                             }
 229
 230
 231                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 232                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
 233                             builder.triggerOnlyOnComment()
 234                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
 235                             builder.triggerForBranch(branch)
 236                             builder.emitTrigger(newJob)
 237                         }
 238                         else if (opt_level == 'full_opt' && pgo_optimized) {
 239                             // Set a push trigger
 240                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 241                             builder.emitTrigger(newJob)
 242                         }
 243                         else {
 244                             // Set periodic trigger
 245                             Utilities.addPeriodicTrigger(newJob, '@daily')
 246                         }
 247                     }
 248                 }
 249             }
 250         }
 251     }
 252 }
 253
 254 def static getFullPerfJobName(def project, def os, def isPR) {
 255     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 256 }
 257
 258 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 259 [true, false].each { isPR ->
 260     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 261     def architecture = 'x64'
 262     def configuration = 'Release'
 263
 264     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 265     ['RHEL7.2'].each { os ->
 266         def newBuildJob = job(fullBuildJobName) {
 267             steps {
 268                 shell("./build.sh verbose ${architecture} ${configuration}")
 269             }
 270         }
 271         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 272         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 273         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 274     }
 275
 276
 277     // Actual perf testing on the following OSes
 278     def perfOSList = ['Ubuntu16.04']
 279     perfOSList.each { os ->
 280         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 281
 282             label('ubuntu_1604_clr_perf')
 283             wrappers {
 284                 credentialsBinding {
 285                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 286                 }
 287             }
 288
 289             if (isPR) {
 290                 parameters {
 291                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 292                 }
 293             }
 294
 295             parameters {
 296                 // Cap the maximum number of iterations to 21.
 297                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 298                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
 299                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 300             }
 301
 302             def osGroup = getOSGroup(os)
 303             def runType = isPR ? 'private' : 'rolling'
 304             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 305
 306             steps {
 307                 shell("./tests/scripts/perf-prep.sh")
 308                 shell("./init-tools.sh")
 309                 copyArtifacts(fullBuildJobName) {
 310                     includePatterns("bin/**")
 311                     buildSelector {
 312                         buildNumber('\${PRODUCT_BUILD}')
 313                     }
 314                 }
 315                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 316                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 317                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 318                 shell("""./tests/scripts/run-xunit-perf.sh \\
 319                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 320                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 321                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 322                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 323                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 324                 --runType=\"${runType}\" \\
 325                 --benchViewOS=\"${os}\" \\
 326                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 327                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 328                 --uploadToBenchview""")
 329                 shell("mkdir -p bin/toArchive/sandbox/Logs/")
 330                 shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
 331             }
 332         }
 333
 334         def archiveSettings = new ArchivalSettings()
 335         archiveSettings.addFiles('bin/toArchive/**')
 336         archiveSettings.addFiles('machinedata.json')
 337
 338         Utilities.addArchival(newJob, archiveSettings)
 339         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 340
 341         // For perf, we need to keep the run results longer
 342         newJob.with {
 343             // Enable the log rotator
 344             logRotator {
 345                 artifactDaysToKeep(30)
 346                 daysToKeep(30)
 347                 artifactNumToKeep(200)
 348                 numToKeep(200)
 349             }
 350             wrappers {
 351                 timeout {
 352                     absolute(240)
 353                 }
 354             }
 355         }
 356     } // os
 357
 358     def flowJobPerfRunList = perfOSList.collect { os ->
 359         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 360     }
 361     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 362         if (isPR) {
 363             parameters {
 364                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 365             }
 366         }
 367         buildFlow("""
 368 // First, build the bits on RHEL7.2
 369 b = build(params, '${fullBuildJobName}')
 370
 371 // Then, run the perf tests
 372 parallel(
 373     ${flowJobPerfRunList.join(",\n    ")}
 374 )
 375 """)
 376     }
 377
 378     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 379     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 380
 381     if (isPR) {
 382         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 383         builder.setGithubContext("Linux Perf Test Flow")
 384         builder.triggerOnlyOnComment()
 385         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 386         builder.triggerForBranch(branch)
 387         builder.emitTrigger(newFlowJob)
 388     }
 389     else {
 390         // Set a push trigger
 391         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 392         builder.emitTrigger(newFlowJob)
 393     }
 394
 395 } // isPR
 396
 397 def static getFullThroughputJobName(def project, def os, def isPR) {
 398     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 399 }
 400
 401 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 402 [true, false].each { isPR ->
 403     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 404     def architecture = 'x64'
 405     def configuration = 'Release'
 406
 407     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 408     ['RHEL7.2'].each { os ->
 409         def newBuildJob = job(fullBuildJobName) {
 410             steps {
 411                 shell("./build.sh verbose ${architecture} ${configuration}")
 412             }
 413         }
 414         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 415         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 416         Utilities.addArchival(newBuildJob, "bin/Product/**")
 417     }
 418
 419     // Actual perf testing on the following OSes
 420     def throughputOSList = ['Ubuntu14.04']
 421     def throughputOptLevelList = ['full_opt', 'min_opt']
 422
 423     def throughputOSOptLevelList = []
 424
 425     throughputOSList.each { os ->
 426         throughputOptLevelList.each { opt_level ->
 427             throughputOSOptLevelList.add("${os}_${opt_level}")
 428         }
 429     }
 430
 431     throughputOSList.each { os ->
 432         throughputOptLevelList.each { opt_level ->
 433             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 434
 435                 label('ubuntu_1604_clr_perf')
 436                     wrappers {
 437                         credentialsBinding {
 438                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 439                         }
 440                     }
 441
 442                 if (isPR) {
 443                     parameters {
 444                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 445                     }
 446                 }
 447
 448                 parameters {
 449                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 450                 }
 451
 452                 def osGroup = getOSGroup(os)
 453                 def runType = isPR ? 'private' : 'rolling'
 454                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 455
 456                 steps {
 457                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 458                     shell("./init-tools.sh")
 459                     copyArtifacts(fullBuildJobName) {
 460                         includePatterns("bin/Product/**")
 461                         buildSelector {
 462                             buildNumber('\${PRODUCT_BUILD}')
 463                         }
 464                     }
 465                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 466                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
 467                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 468                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 469                     -arch \"${architecture}\" \\
 470                     -os \"${os}\" \\
 471                     -configuration \"${configuration}\" \\
 472                     -opt_level \"${opt_level}\" \\
 473                     -clr_root \"\${WORKSPACE}\" \\
 474                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 475                     -run_type \"${runType}\" \\
 476                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 477                 }
 478             }
 479
 480             // Save machinedata.json to /artifact/bin/ Jenkins dir
 481             def archiveSettings = new ArchivalSettings()
 482             archiveSettings.addFiles('throughput-*.csv')
 483             archiveSettings.addFiles('machinedata.json')
 484             Utilities.addArchival(newJob, archiveSettings)
 485
 486             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 487
 488             // For perf, we need to keep the run results longer
 489             newJob.with {
 490                 // Enable the log rotator
 491                 logRotator {
 492                     artifactDaysToKeep(7)
 493                     daysToKeep(300)
 494                     artifactNumToKeep(25)
 495                     numToKeep(1000)
 496                 }
 497             }
 498         } // opt_level
 499     } // os
 500
 501     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 502         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 503     }
 504     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 505         if (isPR) {
 506             parameters {
 507                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 508             }
 509         }
 510         buildFlow("""
 511 // First, build the bits on RHEL7.2
 512 b = build(params, '${fullBuildJobName}')
 513
 514 // Then, run the perf tests
 515 parallel(
 516     ${flowJobTPRunList.join(",\n    ")}
 517 )
 518 """)
 519     }
 520
 521     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 522     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 523
 524     if (isPR) {
 525         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 526         builder.setGithubContext("Linux Throughput Perf Test Flow")
 527         builder.triggerOnlyOnComment()
 528         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 529         builder.triggerForBranch(branch)
 530         builder.emitTrigger(newFlowJob)
 531     }
 532     else {
 533         // Set a push trigger
 534         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 535         builder.emitTrigger(newFlowJob)
 536     }
 537
 538 } // isPR
 539
 540 // Setup CoreCLR-Scenarios tests
 541 [true, false].each { isPR ->
 542     ['Windows_NT'].each { os ->
 543         ['x64', 'x86'].each { arch ->
 544             ['ryujit'].each { jit ->
 545                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
 546                     def architecture = arch
 547                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 548
 549                         def testEnv = ""
 550
 551                         // Set the label.
 552                         label('windows_server_2016_clr_perf')
 553                         wrappers {
 554                             credentialsBinding {
 555                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 556                             }
 557                         }
 558
 559                         if (isPR) {
 560                             parameters {
 561                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 562                             }
 563                         }
 564
 565                         parameters {
 566                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 567                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 568                         }
 569
 570                         def configuration = 'Release'
 571                         def runType = isPR ? 'private' : 'rolling'
 572                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 573                         def uploadString = '-uploadToBenchview'
 574
 575                         steps {
 576                             // Batch
 577                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 578                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 579                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 580
 581                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 582                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 583                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 584                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 585                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 586                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 587                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 588                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 589                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 590
 591                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 592
 593                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
 594
 595                             // Profile=Off
 596                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
 597
 598                             // Profile=On
 599                             if (opt_level != 'min_opt') {
 600                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
 601                             }
 602                         }
 603                     }
 604
 605                     def archiveSettings = new ArchivalSettings()
 606                     archiveSettings.addFiles('bin/sandbox_logs/**')
 607                     archiveSettings.addFiles('machinedata.json')
 608
 609                     Utilities.addArchival(newJob, archiveSettings)
 610                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 611
 612                     newJob.with {
 613                         logRotator {
 614                             artifactDaysToKeep(30)
 615                             daysToKeep(30)
 616                             artifactNumToKeep(200)
 617                             numToKeep(200)
 618                         }
 619                         wrappers {
 620                             timeout {
 621                                 absolute(240)
 622                             }
 623                         }
 624                     }
 625
 626                     if (isPR) {
 627                         def opts = ""
 628                         if (opt_level == 'min_opt') {
 629                             opts = '\\W+min_opts'
 630                         }
 631                         def jitt = ""
 632                         if (jit != 'ryujit') {
 633                             jitt = "\\W+${jit}"
 634                         }
 635
 636                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 637                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
 638                         builder.triggerOnlyOnComment()
 639                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
 640                         builder.triggerForBranch(branch)
 641                         builder.emitTrigger(newJob)
 642                     }
 643                     else if (opt_level == 'full_opt') {
 644                         // Set a push trigger
 645                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 646                         builder.emitTrigger(newJob)
 647                     }
 648                     else {
 649                         // Set periodic trigger
 650                         Utilities.addPeriodicTrigger(newJob, '@daily')
 651                     }
 652                 }
 653             }
 654         }
 655     }
 656 }
 657
 658 // Setup size-on-disk test
 659 ['Windows_NT'].each { os ->
 660     ['x64', 'x86'].each { arch ->
 661         def architecture = arch
 662         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
 663
 664             wrappers {
 665                 credentialsBinding {
 666                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 667                 }
 668             }
 669
 670             def channel = 'master'
 671             def configuration = 'Release'
 672             def runType = 'rolling'
 673             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
 674             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
 675             def coreRoot = "${testBin}\\Tests\\Core_Root"
 676             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
 677
 678             steps {
 679                 // Install nuget and get BenchView tools
 680                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 681                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 682                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 683
 684                 // Generate submission metadata for BenchView
 685                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 686                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 687                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 688                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 689                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 690                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 691                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 692
 693                 // Generate machine data from BenchView
 694                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
 695
 696                 // Build CoreCLR and gnerate test layout
 697                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 698                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 699
 700                 // Run the size on disk benchmark
 701                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
 702
 703                 // From sodbench.csv, create measurment.json, then submission.json
 704                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
 705                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
 706
 707                 // If this is a PR, upload submission.json
 708                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
 709             }
 710         }
 711
 712         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 713
 714         def archiveSettings = new ArchivalSettings()
 715         archiveSettings.addFiles('bin/toArchive/**')
 716         archiveSettings.addFiles('machinedata.json')
 717
 718         Utilities.addArchival(newJob, archiveSettings)
 719         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
 720
 721         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
 722         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
 723
 724         newJob.with {
 725             logRotator {
 726                 artifactDaysToKeep(30)
 727                 daysToKeep(30)
 728                 artifactNumToKeep(200)
 729                 numToKeep(200)
 730             }
 731             wrappers {
 732                 timeout {
 733                     absolute(240)
 734                 }
 735             }
 736         }
 737     }
 738 }
 739
 740 // Setup IlLink tests
 741 [true, false].each { isPR ->
 742     ['Windows_NT'].each { os ->
 743         ['x64'].each { arch ->
 744             ['ryujit'].each { jit ->
 745                 ['full_opt'].each { opt_level ->
 746                     def architecture = arch
 747                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
 748
 749                         def testEnv = ""
 750                         wrappers {
 751                             credentialsBinding {
 752                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 753                             }
 754                         }
 755
 756                         if (isPR) {
 757                             parameters {
 758                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 759                             }
 760                         }
 761
 762                         parameters {
 763                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 764                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 765                         }
 766
 767                         def configuration = 'Release'
 768                         def runType = isPR ? 'private' : 'rolling'
 769                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 770                         def uploadString = '-uploadToBenchview'
 771
 772                         steps {
 773                             // Batch
 774                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 775                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 776                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 777
 778                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 779                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 780                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 781                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 782                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
 783                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
 784                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 785                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 786                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 787
 788                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 789
 790                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
 791
 792                             // Scenario: ILLink
 793                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
 794                         }
 795                     }
 796
 797                     def archiveSettings = new ArchivalSettings()
 798                     archiveSettings.addFiles('bin/sandbox_logs/**')
 799                     archiveSettings.addFiles('machinedata.json')
 800
 801                     // Set the label (currently we are only measuring size, therefore we are running on VM).
 802                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 803                     Utilities.addArchival(newJob, archiveSettings)
 804                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 805
 806                     newJob.with {
 807                         logRotator {
 808                             artifactDaysToKeep(30)
 809                             daysToKeep(30)
 810                             artifactNumToKeep(200)
 811                             numToKeep(200)
 812                         }
 813                         wrappers {
 814                             timeout {
 815                                 absolute(240)
 816                             }
 817                         }
 818                     }
 819
 820                     if (isPR) {
 821                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 822                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
 823                         builder.triggerOnlyOnComment()
 824                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
 825                         builder.triggerForBranch(branch)
 826                         builder.emitTrigger(newJob)
 827                     }
 828                     else {
 829                         // Set a push trigger
 830                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 831                         builder.emitTrigger(newJob)
 832                     }
 833                 }
 834             }
 835         }
 836     }
 837 }
 838
 839 Utilities.createHelperJob(this, project, branch,
 840     "Welcome to the ${project} Perf help",
 841     "Have a nice day!")