perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 def architecture = arch
  34                 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_smoketest" : "perf_perflab_${os}_${arch}"
  35
  36                 if (arch == 'x86')
  37                 {
  38                     testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  39                 }
  40
  41                 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  42                     // Set the label.
  43                     label('windows_clr_perf')
  44                     wrappers {
  45                         credentialsBinding {
  46                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  47                         }
  48                     }
  49
  50                     if (isPR)
  51                     {
  52                         parameters
  53                         {
  54                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  55                         }
  56                     }
  57                     if (isSmoketest)
  58                     {
  59                         parameters
  60                         {
  61                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  62                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  63                         }
  64                     }
  65                     else
  66                     {
  67                         parameters
  68                         {
  69                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  70                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  71                         }
  72                     }
  73                     def configuration = 'Release'
  74                     def runType = isPR ? 'private' : 'rolling'
  75                     def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  76                     def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  77
  78                     steps {
  79                         // Batch
  80
  81                         batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  82                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  83                         batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  84                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  85                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  86                         batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  87                         "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  88                         "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
  89                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
  90                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  91                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  92                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  93
  94                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  95
  96                         // Run with just stopwatch: Profile=Off
  97                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
  98                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
  99
 100                         // Run with the full set of counters enabled: Profile=On
 101                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
 102                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
 103                     }
 104                 }
 105
 106                 if (isSmoketest)
 107                 {
 108                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 109                 }
 110                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 111                 def archiveSettings = new ArchivalSettings()
 112                 archiveSettings.addFiles('Perf-*.xml')
 113                 archiveSettings.addFiles('Perf-*.etl')
 114                 archiveSettings.addFiles('Perf-*.log')
 115                 archiveSettings.addFiles('machinedata.json')
 116                 Utilities.addArchival(newJob, archiveSettings)
 117
 118                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 119
 120                 newJob.with {
 121                     wrappers {
 122                         timeout {
 123                             absolute(240)
 124                         }
 125                     }
 126                 }
 127
 128                 if (isPR) {
 129                     TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 130                     if (isSmoketest)
 131                     {
 132                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests Correctness")
 133                     }
 134                     else
 135                     {
 136                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 137                         builder.triggerOnlyOnComment()
 138                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 139                     }
 140                     builder.triggerForBranch(branch)
 141                     builder.emitTrigger(newJob)
 142                 }
 143                 else {
 144                     // Set a push trigger
 145                     TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 146                     builder.emitTrigger(newJob)
 147                 }
 148             }
 149         }
 150     }
 151 }
 152
 153 // Setup throughput perflab tests runs
 154 [true, false].each { isPR ->
 155     ['Windows_NT'].each { os ->
 156         ['x64', 'x86'].each { arch ->
 157             ['full_opt', 'min_opt'].each { opt_level ->
 158                 def architecture = arch
 159
 160                 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}", isPR)) {
 161                     // Set the label.
 162                     label('windows_clr_perf')
 163                     wrappers {
 164                         credentialsBinding {
 165                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 166                         }
 167                     }
 168
 169                     if (isPR)
 170                     {
 171                         parameters
 172                         {
 173                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 174                         }
 175                     }
 176                     def configuration = 'Release'
 177                     def runType = isPR ? 'private' : 'rolling'
 178                     def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 179
 180                     steps {
 181                         // Batch
 182
 183                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 184                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 185                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 186                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 187                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 188                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 189                         batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 190                         "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 191                         "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 192                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 193                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 194                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 195                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 196                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 197                         batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 198                     }
 199                 }
 200
 201                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 202                 def archiveSettings = new ArchivalSettings()
 203                 archiveSettings.addFiles('throughput-*.csv')
 204                 Utilities.addArchival(newJob, archiveSettings)
 205
 206                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 207
 208                 if (isPR) {
 209                     def opts = ""
 210                     if (opt_level == 'min_opts')
 211                     {
 212                         opts = '\\W+min_opts'
 213                     }
 214                     TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 215                     builder.setGithubContext("${os} ${arch} ${opt_level} CoreCLR Throughput Perf Tests")
 216                     builder.triggerOnlyOnComment()
 217                     builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}\\W+throughput.*")
 218                     builder.triggerForBranch(branch)
 219                     builder.emitTrigger(newJob)
 220                 }
 221                 else {
 222                     // Set a push trigger
 223                     TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 224                     builder.emitTrigger(newJob)
 225                 }
 226             }
 227         }
 228     }
 229 }
 230
 231 def static getFullPerfJobName(def project, def os, def isPR) {
 232     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 233 }
 234
 235 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 236 [true, false].each { isPR ->
 237     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 238     def architecture = 'x64'
 239     def configuration = 'Release'
 240
 241     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 242     ['RHEL7.2'].each { os ->
 243         def newBuildJob = job(fullBuildJobName) {
 244             steps {
 245                 shell("./build.sh verbose ${architecture} ${configuration}")
 246             }
 247         }
 248         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 249         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 250         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 251     }
 252
 253     // Actual perf testing on the following OSes
 254     def perfOSList = ['Ubuntu14.04']
 255     perfOSList.each { os ->
 256         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 257
 258             label('linux_clr_perf')
 259             wrappers {
 260                 credentialsBinding {
 261                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 262                 }
 263             }
 264
 265             if (isPR)
 266             {
 267                 parameters
 268                 {
 269                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 270                 }
 271             }
 272
 273             parameters {
 274                 // Cap the maximum number of iterations to 21.
 275                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
 276                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
 277                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 278             }
 279
 280             def osGroup = getOSGroup(os)
 281             def runType = isPR ? 'private' : 'rolling'
 282             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 283
 284             steps {
 285                 shell("./tests/scripts/perf-prep.sh")
 286                 shell("./init-tools.sh")
 287                 copyArtifacts(fullBuildJobName) {
 288                     includePatterns("bin/**")
 289                     buildSelector {
 290                         buildNumber('\${PRODUCT_BUILD}')
 291                     }
 292                 }
 293                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 294                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 295                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 296                 shell("""./tests/scripts/run-xunit-perf.sh \\
 297                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 298                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 299                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 300                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 301                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 302                 --runType=\"${runType}\" \\
 303                 --benchViewOS=\"${os}\" \\
 304                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 305                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 306                 --uploadToBenchview""")
 307             }
 308         }
 309
 310         // Save machinedata.json to /artifact/bin/ Jenkins dir
 311         def archiveSettings = new ArchivalSettings()
 312         archiveSettings.addFiles('Perf-*.log')
 313         archiveSettings.addFiles('Perf-*.xml')
 314         archiveSettings.addFiles('machinedata.json')
 315         Utilities.addArchival(newJob, archiveSettings)
 316
 317         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 318
 319         // For perf, we need to keep the run results longer
 320         newJob.with {
 321             // Enable the log rotator
 322             logRotator {
 323                 artifactDaysToKeep(7)
 324                 daysToKeep(300)
 325                 artifactNumToKeep(25)
 326                 numToKeep(1000)
 327             }
 328         }
 329     } // os
 330
 331     def flowJobPerfRunList = perfOSList.collect { os ->
 332         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 333     }
 334     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 335         if (isPR) {
 336             parameters {
 337                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 338             }
 339         }
 340         buildFlow("""
 341 // First, build the bits on RHEL7.2
 342 b = build(params, '${fullBuildJobName}')
 343
 344 // Then, run the perf tests
 345 parallel(
 346     ${flowJobPerfRunList.join(",\n    ")}
 347 )
 348 """)
 349     }
 350
 351     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 352     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 353
 354     if (isPR) {
 355         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 356         builder.setGithubContext("Linux Perf Test Flow")
 357         builder.triggerOnlyOnComment()
 358         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 359         builder.triggerForBranch(branch)
 360         builder.emitTrigger(newFlowJob)
 361     }
 362     else {
 363         // Set a push trigger
 364         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 365         builder.emitTrigger(newFlowJob)
 366     }
 367
 368 } // isPR
 369
 370 def static getFullThroughputJobName(def project, def os, def isPR) {
 371     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 372 }
 373
 374 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 375 [true, false].each { isPR ->
 376     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 377     def architecture = 'x64'
 378     def configuration = 'Release'
 379
 380     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 381     ['RHEL7.2'].each { os ->
 382         def newBuildJob = job(fullBuildJobName) {
 383             steps {
 384                 shell("./build.sh verbose ${architecture} ${configuration}")
 385             }
 386         }
 387         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 388         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 389         Utilities.addArchival(newBuildJob, "bin/Product/**")
 390     }
 391
 392     // Actual perf testing on the following OSes
 393     def throughputOSList = ['Ubuntu14.04']
 394     def throughputOptLevelList = ['full_opt', 'min_opt']
 395
 396     def throughputOSOptLevelList = []
 397
 398     throughputOSList.each { os ->
 399         throughputOptLevelList.each { opt_level ->
 400             throughputOSOptLevelList.add("${os}_${opt_level}")
 401         }
 402     }
 403
 404     throughputOSList.each { os ->
 405         throughputOptLevelList.each { opt_level ->
 406             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 407
 408                 label('linux_clr_perf')
 409                     wrappers {
 410                         credentialsBinding {
 411                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 412                         }
 413                     }
 414
 415                 if (isPR)
 416                 {
 417                     parameters
 418                     {
 419                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 420                     }
 421                 }
 422
 423                 parameters {
 424                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 425                 }
 426
 427                 def osGroup = getOSGroup(os)
 428                 def runType = isPR ? 'private' : 'rolling'
 429                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 430
 431                 steps {
 432                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 433                     shell("./init-tools.sh")
 434                     copyArtifacts(fullBuildJobName) {
 435                         includePatterns("bin/Product/**")
 436                         buildSelector {
 437                             buildNumber('\${PRODUCT_BUILD}')
 438                         }
 439                     }
 440                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 441                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 442                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 443                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 444                     -arch \"${architecture}\" \\
 445                     -os \"${os}\" \\
 446                     -configuration \"${configuration}\" \\
 447                     -opt_level \"${opt_level}\" \\
 448                     -clr_root \"\${WORKSPACE}\" \\
 449                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 450                     -run_type \"${runType}\" \\
 451                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 452                 }
 453             }
 454
 455             // Save machinedata.json to /artifact/bin/ Jenkins dir
 456             def archiveSettings = new ArchivalSettings()
 457             archiveSettings.addFiles('throughput-*.csv')
 458             archiveSettings.addFiles('machinedata.json')
 459             Utilities.addArchival(newJob, archiveSettings)
 460
 461             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 462
 463             // For perf, we need to keep the run results longer
 464             newJob.with {
 465                 // Enable the log rotator
 466                 logRotator {
 467                     artifactDaysToKeep(7)
 468                     daysToKeep(300)
 469                     artifactNumToKeep(25)
 470                     numToKeep(1000)
 471                 }
 472             }
 473         } // opt_level
 474     } // os
 475
 476     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 477         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 478     }
 479     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 480         if (isPR) {
 481             parameters {
 482                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 483             }
 484         }
 485         buildFlow("""
 486 // First, build the bits on RHEL7.2
 487 b = build(params, '${fullBuildJobName}')
 488
 489 // Then, run the perf tests
 490 parallel(
 491     ${flowJobTPRunList.join(",\n    ")}
 492 )
 493 """)
 494     }
 495
 496     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 497     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 498
 499     if (isPR) {
 500         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 501         builder.setGithubContext("Linux Throughput Perf Test Flow")
 502         builder.triggerOnlyOnComment()
 503         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 504         builder.triggerForBranch(branch)
 505         builder.emitTrigger(newFlowJob)
 506     }
 507     else {
 508         // Set a push trigger
 509         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 510         builder.emitTrigger(newFlowJob)
 511     }
 512
 513 } // isPR
 514
 515 // Setup CoreCLR-Scenarios tests
 516 [true, false].each { isPR ->
 517     ['Windows_NT'].each { os ->
 518         ['x64', 'x86'].each { arch ->
 519             def architecture = arch
 520             def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}", isPR)) {
 521                 // Set the label.
 522                 label('windows_clr_perf')
 523                 wrappers {
 524                     credentialsBinding {
 525                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 526                     }
 527                 }
 528
 529                 if (isPR)
 530                 {
 531                     parameters
 532                     {
 533                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 534                     }
 535                 }
 536
 537                 parameters
 538                 {
 539                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 540                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 541                 }
 542                 def configuration = 'Release'
 543                 def runType = isPR ? 'private' : 'rolling'
 544                 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 545                 def uploadString = '-uploadToBenchview'
 546
 547                 steps {
 548                     // Batch
 549                     batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 550                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 551                     batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 552
 553                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 554                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 555                     batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 556                     "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 557                     "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 558                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
 559                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 560                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 561                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 562
 563                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 564
 565                     // Scenario: JitBench
 566                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -scenarioTest -group CoreCLR-Scenarios")
 567
 568                     // Scenario: ILLink
 569                     if (arch == 'x64') {
 570                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -nowarmup -runtype ${runType} -scenarioTest -group ILLink")
 571                     }
 572                 }
 573              }
 574
 575              // Save machinedata.json to /artifact/bin/ Jenkins dir
 576             def archiveSettings = new ArchivalSettings()
 577             archiveSettings.addFiles('Perf-*.xml')
 578             archiveSettings.addFiles('Perf-*.log')
 579             archiveSettings.addFiles('machinedata.json')
 580             Utilities.addArchival(newJob, archiveSettings)
 581
 582             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 583
 584             newJob.with {
 585                 wrappers {
 586                     timeout {
 587                         absolute(240)
 588                     }
 589                 }
 590             }
 591
 592             if (isPR) {
 593                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 594                 builder.setGithubContext("${os} ${arch} Performance Scenarios Tests")
 595                 builder.triggerOnlyOnComment()
 596                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf\\W+scenarios.*")
 597                 builder.triggerForBranch(branch)
 598                 builder.emitTrigger(newJob)
 599             }
 600             else {
 601                 // Set a push trigger
 602                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 603                 builder.emitTrigger(newJob)
 604             }
 605         }
 606     }
 607 }
 608
 609 Utilities.createHelperJob(this, project, branch,
 610     "Welcome to the ${project} Perf help",
 611     "Have a nice day!")