perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 def architecture = arch
  34                 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_smoketest" : "perf_perflab_${os}_${arch}"
  35
  36                 if (arch == 'x86') {
  37                     testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  38                 }
  39
  40                 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  41                     // Set the label.
  42                     label('windows_clr_perf')
  43                     wrappers {
  44                         credentialsBinding {
  45                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  46                         }
  47                     }
  48
  49                     if (isPR) {
  50                         parameters {
  51                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  52                         }
  53                     }
  54
  55                     if (isSmoketest) {
  56                         parameters {
  57                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  58                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  59                         }
  60                     }
  61                     else {
  62                         parameters {
  63                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  64                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  65                         }
  66                     }
  67
  68                     def configuration = 'Release'
  69                     def runType = isPR ? 'private' : 'rolling'
  70                     def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  71                     def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  72
  73                     steps {
  74                         // Batch
  75
  76                         batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  77                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  78                         batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  79                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  80                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  81                         batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  82                         "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  83                         "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
  84                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
  85                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  86                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  87                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  88
  89                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  90
  91                         def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
  92
  93                         // Run with just stopwatch: Profile=Off
  94                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
  95                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
  96
  97                         // Run with the full set of counters enabled: Profile=On
  98                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
  99                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 100                     }
 101                 }
 102
 103                 if (isSmoketest) {
 104                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 105                 }
 106
 107                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 108                 def archiveSettings = new ArchivalSettings()
 109                 archiveSettings.addFiles('.\\bin\\sandbox\\Logs\\Perf-*.xml')
 110                 archiveSettings.addFiles('.\\bin\\sandbox\\Logs\\Perf-*.etl')
 111                 archiveSettings.addFiles('.\\bin\\sandbox\\Logs\\Perf-*.log')
 112                 archiveSettings.addFiles('machinedata.json')
 113                 Utilities.addArchival(newJob, archiveSettings)
 114
 115                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 116
 117                 newJob.with {
 118                     wrappers {
 119                         timeout {
 120                             absolute(240)
 121                         }
 122                     }
 123                 }
 124
 125                 if (isPR) {
 126                     TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 127                     if (isSmoketest)
 128                     {
 129                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests Correctness")
 130                     }
 131                     else
 132                     {
 133                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 134                         builder.triggerOnlyOnComment()
 135                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 136                     }
 137                     builder.triggerForBranch(branch)
 138                     builder.emitTrigger(newJob)
 139                 }
 140                 else {
 141                     // Set a push trigger
 142                     TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 143                     builder.emitTrigger(newJob)
 144                 }
 145             }
 146         }
 147     }
 148 }
 149
 150 // Setup throughput perflab tests runs
 151 [true, false].each { isPR ->
 152     ['Windows_NT'].each { os ->
 153         ['x64', 'x86'].each { arch ->
 154             ['full_opt', 'min_opt'].each { opt_level ->
 155                 def architecture = arch
 156
 157                 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}", isPR)) {
 158                     // Set the label.
 159                     label('windows_clr_perf')
 160                     wrappers {
 161                         credentialsBinding {
 162                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 163                         }
 164                     }
 165
 166                     if (isPR) {
 167                         parameters {
 168                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 169                         }
 170                     }
 171
 172                     def configuration = 'Release'
 173                     def runType = isPR ? 'private' : 'rolling'
 174                     def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 175
 176                     steps {
 177                         // Batch
 178                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 179                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 180                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 181                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 182                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 183                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 184                         batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 185                         "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 186                         "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 187                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 188                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 189                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 190                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 191                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 192                         batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 193                     }
 194                 }
 195
 196                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 197                 def archiveSettings = new ArchivalSettings()
 198                 archiveSettings.addFiles('throughput-*.csv')
 199                 Utilities.addArchival(newJob, archiveSettings)
 200
 201                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 202
 203                 if (isPR) {
 204                     def opts = ""
 205                     if (opt_level == 'min_opts')
 206                     {
 207                         opts = '\\W+min_opts'
 208                     }
 209                     TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 210                     builder.setGithubContext("${os} ${arch} ${opt_level} CoreCLR Throughput Perf Tests")
 211                     builder.triggerOnlyOnComment()
 212                     builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}\\W+throughput.*")
 213                     builder.triggerForBranch(branch)
 214                     builder.emitTrigger(newJob)
 215                 }
 216                 else {
 217                     // Set a push trigger
 218                     TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 219                     builder.emitTrigger(newJob)
 220                 }
 221             }
 222         }
 223     }
 224 }
 225
 226 def static getFullPerfJobName(def project, def os, def isPR) {
 227     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 228 }
 229
 230 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 231 [true, false].each { isPR ->
 232     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 233     def architecture = 'x64'
 234     def configuration = 'Release'
 235
 236     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 237     ['RHEL7.2'].each { os ->
 238         def newBuildJob = job(fullBuildJobName) {
 239             steps {
 240                 shell("./build.sh verbose ${architecture} ${configuration}")
 241             }
 242         }
 243         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 244         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 245         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 246     }
 247
 248
 249     // Actual perf testing on the following OSes
 250     def perfOSList = ['Ubuntu14.04']
 251     perfOSList.each { os ->
 252         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 253
 254             label('linux_clr_perf')
 255             wrappers {
 256                 credentialsBinding {
 257                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 258                 }
 259             }
 260
 261             if (isPR) {
 262                 parameters {
 263                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 264                 }
 265             }
 266
 267             parameters {
 268                 // Cap the maximum number of iterations to 21.
 269                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
 270                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
 271                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 272             }
 273
 274             def osGroup = getOSGroup(os)
 275             def runType = isPR ? 'private' : 'rolling'
 276             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 277
 278             steps {
 279                 shell("./tests/scripts/perf-prep.sh")
 280                 shell("./init-tools.sh")
 281                 copyArtifacts(fullBuildJobName) {
 282                     includePatterns("bin/**")
 283                     buildSelector {
 284                         buildNumber('\${PRODUCT_BUILD}')
 285                     }
 286                 }
 287                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 288                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 289                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 290                 shell("""./tests/scripts/run-xunit-perf.sh \\
 291                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 292                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 293                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 294                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 295                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 296                 --runType=\"${runType}\" \\
 297                 --benchViewOS=\"${os}\" \\
 298                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 299                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 300                 --uploadToBenchview""")
 301             }
 302         }
 303
 304         // Save machinedata.json to /artifact/bin/ Jenkins dir
 305         def archiveSettings = new ArchivalSettings()
 306         archiveSettings.addFiles('./bin/sandbox/Logs/Perf-*.log')
 307         archiveSettings.addFiles('./bin/sandbox/Logs/Perf-*.xml')
 308         archiveSettings.addFiles('machinedata.json')
 309         Utilities.addArchival(newJob, archiveSettings)
 310
 311         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 312
 313         // For perf, we need to keep the run results longer
 314         newJob.with {
 315             // Enable the log rotator
 316             logRotator {
 317                 artifactDaysToKeep(7)
 318                 daysToKeep(300)
 319                 artifactNumToKeep(25)
 320                 numToKeep(1000)
 321             }
 322         }
 323     } // os
 324
 325     def flowJobPerfRunList = perfOSList.collect { os ->
 326         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 327     }
 328     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 329         if (isPR) {
 330             parameters {
 331                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 332             }
 333         }
 334         buildFlow("""
 335 // First, build the bits on RHEL7.2
 336 b = build(params, '${fullBuildJobName}')
 337
 338 // Then, run the perf tests
 339 parallel(
 340     ${flowJobPerfRunList.join(",\n    ")}
 341 )
 342 """)
 343     }
 344
 345     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 346     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 347
 348     if (isPR) {
 349         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 350         builder.setGithubContext("Linux Perf Test Flow")
 351         builder.triggerOnlyOnComment()
 352         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 353         builder.triggerForBranch(branch)
 354         builder.emitTrigger(newFlowJob)
 355     }
 356     else {
 357         // Set a push trigger
 358         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 359         builder.emitTrigger(newFlowJob)
 360     }
 361
 362 } // isPR
 363
 364 def static getFullThroughputJobName(def project, def os, def isPR) {
 365     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 366 }
 367
 368 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 369 [true, false].each { isPR ->
 370     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 371     def architecture = 'x64'
 372     def configuration = 'Release'
 373
 374     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 375     ['RHEL7.2'].each { os ->
 376         def newBuildJob = job(fullBuildJobName) {
 377             steps {
 378                 shell("./build.sh verbose ${architecture} ${configuration}")
 379             }
 380         }
 381         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 382         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 383         Utilities.addArchival(newBuildJob, "bin/Product/**")
 384     }
 385
 386     // Actual perf testing on the following OSes
 387     def throughputOSList = ['Ubuntu14.04']
 388     def throughputOptLevelList = ['full_opt', 'min_opt']
 389
 390     def throughputOSOptLevelList = []
 391
 392     throughputOSList.each { os ->
 393         throughputOptLevelList.each { opt_level ->
 394             throughputOSOptLevelList.add("${os}_${opt_level}")
 395         }
 396     }
 397
 398     throughputOSList.each { os ->
 399         throughputOptLevelList.each { opt_level ->
 400             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
 401
 402                 label('linux_clr_perf')
 403                     wrappers {
 404                         credentialsBinding {
 405                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 406                         }
 407                     }
 408
 409                 if (isPR)
 410                 {
 411                     parameters
 412                     {
 413                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
 414                     }
 415                 }
 416
 417                 parameters {
 418                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 419                 }
 420
 421                 def osGroup = getOSGroup(os)
 422                 def runType = isPR ? 'private' : 'rolling'
 423                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 424
 425                 steps {
 426                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
 427                     shell("./init-tools.sh")
 428                     copyArtifacts(fullBuildJobName) {
 429                         includePatterns("bin/Product/**")
 430                         buildSelector {
 431                             buildNumber('\${PRODUCT_BUILD}')
 432                         }
 433                     }
 434                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 435                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 436                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 437                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 438                     -arch \"${architecture}\" \\
 439                     -os \"${os}\" \\
 440                     -configuration \"${configuration}\" \\
 441                     -opt_level \"${opt_level}\" \\
 442                     -clr_root \"\${WORKSPACE}\" \\
 443                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 444                     -run_type \"${runType}\" \\
 445                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 446                 }
 447             }
 448
 449             // Save machinedata.json to /artifact/bin/ Jenkins dir
 450             def archiveSettings = new ArchivalSettings()
 451             archiveSettings.addFiles('throughput-*.csv')
 452             archiveSettings.addFiles('machinedata.json')
 453             Utilities.addArchival(newJob, archiveSettings)
 454
 455             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 456
 457             // For perf, we need to keep the run results longer
 458             newJob.with {
 459                 // Enable the log rotator
 460                 logRotator {
 461                     artifactDaysToKeep(7)
 462                     daysToKeep(300)
 463                     artifactNumToKeep(25)
 464                     numToKeep(1000)
 465                 }
 466             }
 467         } // opt_level
 468     } // os
 469
 470     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
 471         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 472     }
 473     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 474         if (isPR) {
 475             parameters {
 476                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 477             }
 478         }
 479         buildFlow("""
 480 // First, build the bits on RHEL7.2
 481 b = build(params, '${fullBuildJobName}')
 482
 483 // Then, run the perf tests
 484 parallel(
 485     ${flowJobTPRunList.join(",\n    ")}
 486 )
 487 """)
 488     }
 489
 490     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 491     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 492
 493     if (isPR) {
 494         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 495         builder.setGithubContext("Linux Throughput Perf Test Flow")
 496         builder.triggerOnlyOnComment()
 497         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 498         builder.triggerForBranch(branch)
 499         builder.emitTrigger(newFlowJob)
 500     }
 501     else {
 502         // Set a push trigger
 503         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 504         builder.emitTrigger(newFlowJob)
 505     }
 506
 507 } // isPR
 508
 509 // Setup CoreCLR-Scenarios tests
 510 [true, false].each { isPR ->
 511     ['Windows_NT'].each { os ->
 512         ['x64', 'x86'].each { arch ->
 513             def architecture = arch
 514             def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}", isPR)) {
 515                 // Set the label.
 516                 label('windows_clr_perf')
 517                 wrappers {
 518                     credentialsBinding {
 519                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 520                     }
 521                 }
 522
 523                 if (isPR) {
 524                     parameters {
 525                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 526                     }
 527                 }
 528
 529                 parameters {
 530                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 531                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 532                 }
 533
 534                 def configuration = 'Release'
 535                 def runType = isPR ? 'private' : 'rolling'
 536                 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 537                 def uploadString = '-uploadToBenchview'
 538
 539                 steps {
 540                     // Batch
 541                     batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 542                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 543                     batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 544
 545                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 546                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 547                     batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 548                     "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 549                     "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 550                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
 551                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 552                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 553                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 554
 555                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 556
 557                     def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -scenarioTest"
 558                     def failedOutputLogFilename = "run-xunit-perf-scenario.log"
 559
 560                     // Using a sentinel file to
 561                     batchFile("if exist \"${failedOutputLogFilename}\" del /q /f \"${failedOutputLogFilename}\"")
 562                     batchFile("if exist \"${failedOutputLogFilename}\" (echo [ERROR] Failed to delete previously created \"${failedOutputLogFilename}\" file.& exit /b 1)")
 563
 564                     // Scenario: JitBench
 565                     batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios || (echo [ERROR] JitBench failed. 1>>\"${failedOutputLogFilename}\"& exit /b 0)")
 566
 567                     // Scenario: ILLink
 568                     if (arch == 'x64') {
 569                         batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup || (echo [ERROR] IlLink failed. 1>>\"${failedOutputLogFilename}\"& exit /b 0)")
 570                     }
 571
 572                     batchFile("if exist \"${failedOutputLogFilename}\" (type \"${failedOutputLogFilename}\"& exit /b 1)")
 573                 }
 574             }
 575
 576              // Save machinedata.json to /artifact/bin/ Jenkins dir
 577             def archiveSettings = new ArchivalSettings()
 578             archiveSettings.addFiles('.\\bin\\sandbox\\Perf-*.xml')
 579             archiveSettings.addFiles('.\\bin\\sandbox\\Perf-*.log')
 580             archiveSettings.addFiles('machinedata.json')
 581             Utilities.addArchival(newJob, archiveSettings)
 582
 583             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 584
 585             newJob.with {
 586                 wrappers {
 587                     timeout {
 588                         absolute(240)
 589                     }
 590                 }
 591             }
 592
 593             if (isPR) {
 594                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 595                 builder.setGithubContext("${os} ${arch} Performance Scenarios Tests")
 596                 builder.triggerOnlyOnComment()
 597                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf\\W+scenarios.*")
 598                 builder.triggerForBranch(branch)
 599                 builder.emitTrigger(newJob)
 600             }
 601             else {
 602                 // Set a push trigger
 603                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 604                 builder.emitTrigger(newJob)
 605             }
 606         }
 607     }
 608 }
 609
 610 Utilities.createHelperJob(this, project, branch,
 611     "Welcome to the ${project} Perf help",
 612     "Have a nice day!")