perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27
  28 // Setup perflab tests runs
  29 [true, false].each { isPR ->
  30     ['Windows_NT'].each { os ->
  31         ['x64', 'x86'].each { arch ->
  32             [true, false].each { isSmoketest ->
  33                 def architecture = arch
  34                 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_smoketest" : "perf_perflab_${os}_${arch}"
  35
  36                 if (arch == 'x86jit32')
  37                 {
  38                     architecture = 'x86'
  39                     testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
  40                 }
  41                 else if (arch == 'x86')
  42                 {
  43                     testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  44                 }
  45
  46                 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  47                     // Set the label.
  48                     label('windows_clr_perf')
  49                     wrappers {
  50                         credentialsBinding {
  51                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  52                         }
  53                     }
  54
  55                     if (isPR)
  56                     {
  57                         parameters
  58                         {
  59                             stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  60                         }
  61                     }
  62                     if (isSmoketest)
  63                     {
  64                         parameters
  65                         {
  66                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  67                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  68                         }
  69                     }
  70                     else
  71                     {
  72                         parameters
  73                         {
  74                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  75                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  76                         }
  77                     }
  78                     def configuration = 'Release'
  79                     def runType = isPR ? 'private' : 'rolling'
  80                     def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  81                     def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  82
  83                     steps {
  84                         // Batch
  85
  86                         batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  87                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  88                         batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  89                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  90                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  91                         batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
  92                         "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
  93                         "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
  94                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
  95                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  96                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  97                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  98
  99                         if (arch == 'x86jit32')
 100                         {
 101                             // Download package and copy compatjit into Core_Root
 102                             batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
 103                             "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
 104                         }
 105
 106                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 107
 108                         // Run with just stopwatch: Profile=Off
 109                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
 110                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
 111
 112                         // Run with the full set of counters enabled: Profile=On
 113                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
 114                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
 115                     }
 116                 }
 117
 118                 if (isSmoketest)
 119                 {
 120                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 121                 }
 122                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 123                 def archiveSettings = new ArchivalSettings()
 124                 archiveSettings.addFiles('Perf-*.xml')
 125                 archiveSettings.addFiles('Perf-*.etl')
 126                 archiveSettings.addFiles('Perf-*.log')
 127                 archiveSettings.addFiles('machinedata.json')
 128                 Utilities.addArchival(newJob, archiveSettings)
 129
 130                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 131
 132                 newJob.with {
 133                     wrappers {
 134                         timeout {
 135                             absolute(240)
 136                         }
 137                     }
 138                 }
 139
 140                 if (isPR) {
 141                     TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 142                     if (isSmoketest)
 143                     {
 144                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests Correctness")
 145                     }
 146                     else
 147                     {
 148                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 149                         builder.triggerOnlyOnComment()
 150                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 151                     }
 152                     builder.triggerForBranch(branch)
 153                     builder.emitTrigger(newJob)
 154                 }
 155                 else {
 156                     // Set a push trigger
 157                     TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 158                     builder.emitTrigger(newJob)
 159                 }
 160             }
 161         }
 162     }
 163 }
 164
 165 // Setup throughput perflab tests runs
 166 [true, false].each { isPR ->
 167     ['Windows_NT'].each { os ->
 168         ['x64', 'x86'].each { arch ->
 169             def architecture = arch
 170
 171             def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
 172                 // Set the label.
 173                 label('windows_clr_perf')
 174                 wrappers {
 175                     credentialsBinding {
 176                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 177                     }
 178                 }
 179
 180             if (isPR)
 181             {
 182                 parameters
 183                 {
 184                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 185                 }
 186             }
 187             def configuration = 'Release'
 188             def runType = isPR ? 'private' : 'rolling'
 189             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 190
 191                 steps {
 192                     // Batch
 193
 194                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 195                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 196                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 197                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 198                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 199                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 200                     batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 201                     "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 202                     "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 203                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 204                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 205                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 206                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 207                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 208                     batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 209                 }
 210             }
 211
 212             // Save machinedata.json to /artifact/bin/ Jenkins dir
 213             def archiveSettings = new ArchivalSettings()
 214             archiveSettings.addFiles('throughput-*.csv')
 215             Utilities.addArchival(newJob, archiveSettings)
 216
 217             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 218
 219             if (isPR) {
 220                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 221                 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
 222                 builder.triggerOnlyOnComment()
 223                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
 224                 builder.triggerForBranch(branch)
 225                 builder.emitTrigger(newJob)
 226             }
 227             else {
 228                 // Set a push trigger
 229                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 230                 builder.emitTrigger(newJob)
 231             }
 232         }
 233     }
 234 }
 235
 236 def static getFullPerfJobName(def project, def os, def isPR) {
 237     return Utilities.getFullJobName(project, "perf_${os}", isPR)
 238 }
 239
 240 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 241 [true, false].each { isPR ->
 242     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
 243     def architecture = 'x64'
 244     def configuration = 'Release'
 245
 246     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 247     ['RHEL7.2'].each { os ->
 248         def newBuildJob = job(fullBuildJobName) {
 249             steps {
 250                 shell("./build.sh verbose ${architecture} ${configuration}")
 251             }
 252         }
 253         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 254         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 255         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
 256     }
 257
 258     // Actual perf testing on the following OSes
 259     def perfOSList = ['Ubuntu14.04']
 260     perfOSList.each { os ->
 261         def newJob = job(getFullPerfJobName(project, os, isPR)) {
 262
 263             label('linux_clr_perf')
 264             wrappers {
 265                 credentialsBinding {
 266                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 267                 }
 268             }
 269
 270             if (isPR)
 271             {
 272                 parameters
 273                 {
 274                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 275                 }
 276             }
 277
 278             parameters {
 279                 // Cap the maximum number of iterations to 21.
 280                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
 281                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
 282                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 283             }
 284
 285             def osGroup = getOSGroup(os)
 286             def runType = isPR ? 'private' : 'rolling'
 287             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 288
 289             steps {
 290                 shell("./tests/scripts/perf-prep.sh")
 291                 shell("./init-tools.sh")
 292                 copyArtifacts(fullBuildJobName) {
 293                     includePatterns("bin/**")
 294                     buildSelector {
 295                         buildNumber('\${PRODUCT_BUILD}')
 296                     }
 297                 }
 298                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 299                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 300                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 301                 shell("""./tests/scripts/run-xunit-perf.sh \\
 302                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 303                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 304                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 305                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 306                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 307                 --runType=\"${runType}\" \\
 308                 --benchViewOS=\"${os}\" \\
 309                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
 310                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
 311                 --uploadToBenchview""")
 312             }
 313         }
 314
 315         // Save machinedata.json to /artifact/bin/ Jenkins dir
 316         def archiveSettings = new ArchivalSettings()
 317         archiveSettings.addFiles('Perf-*.log')
 318         archiveSettings.addFiles('Perf-*.xml')
 319         archiveSettings.addFiles('machinedata.json')
 320         Utilities.addArchival(newJob, archiveSettings)
 321
 322         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 323
 324         // For perf, we need to keep the run results longer
 325         newJob.with {
 326             // Enable the log rotator
 327             logRotator {
 328                 artifactDaysToKeep(7)
 329                 daysToKeep(300)
 330                 artifactNumToKeep(25)
 331                 numToKeep(1000)
 332             }
 333         }
 334     } // os
 335
 336     def flowJobPerfRunList = perfOSList.collect { os ->
 337         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
 338     }
 339     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
 340         if (isPR) {
 341             parameters {
 342                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 343             }
 344         }
 345         buildFlow("""
 346 // First, build the bits on RHEL7.2
 347 b = build(params, '${fullBuildJobName}')
 348
 349 // Then, run the perf tests
 350 parallel(
 351     ${flowJobPerfRunList.join(",\n    ")}
 352 )
 353 """)
 354     }
 355
 356     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 357     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 358
 359     if (isPR) {
 360         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 361         builder.setGithubContext("Linux Perf Test Flow")
 362         builder.triggerOnlyOnComment()
 363         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
 364         builder.triggerForBranch(branch)
 365         builder.emitTrigger(newFlowJob)
 366     }
 367     else {
 368         // Set a push trigger
 369         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 370         builder.emitTrigger(newFlowJob)
 371     }
 372
 373 } // isPR
 374
 375 def static getFullThroughputJobName(def project, def os, def isPR) {
 376     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
 377 }
 378
 379 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 380 [true, false].each { isPR ->
 381     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
 382     def architecture = 'x64'
 383     def configuration = 'Release'
 384
 385     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
 386     ['RHEL7.2'].each { os ->
 387         def newBuildJob = job(fullBuildJobName) {
 388             steps {
 389                 shell("./build.sh verbose ${architecture} ${configuration}")
 390             }
 391         }
 392         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
 393         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
 394         Utilities.addArchival(newBuildJob, "bin/Product/**")
 395     }
 396
 397     // Actual perf testing on the following OSes
 398     def throughputOSList = ['Ubuntu14.04']
 399     throughputOSList.each { os ->
 400         def newJob = job(getFullThroughputJobName(project, os, isPR)) {
 401
 402             label('linux_clr_perf')
 403                 wrappers {
 404                     credentialsBinding {
 405                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 406                     }
 407                 }
 408
 409             if (isPR)
 410             {
 411                 parameters
 412                 {
 413                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 414                 }
 415             }
 416
 417             parameters {
 418                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
 419             }
 420
 421             def osGroup = getOSGroup(os)
 422             def runType = isPR ? 'private' : 'rolling'
 423             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 424
 425             steps {
 426                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
 427                 shell("./init-tools.sh")
 428                 copyArtifacts(fullBuildJobName) {
 429                     includePatterns("bin/Product/**")
 430                     buildSelector {
 431                         buildNumber('\${PRODUCT_BUILD}')
 432                     }
 433                 }
 434                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 435                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 436                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 437                 shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
 438                 -arch \"${architecture}\" \\
 439                 -os \"${os}\" \\
 440                 -configuration \"${configuration}\" \\
 441                 -clr_root \"\${WORKSPACE}\" \\
 442                 -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
 443                 -run_type \"${runType}\" \\
 444                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 445             }
 446         }
 447
 448         // Save machinedata.json to /artifact/bin/ Jenkins dir
 449         def archiveSettings = new ArchivalSettings()
 450         archiveSettings.addFiles('throughput-*.csv')
 451         archiveSettings.addFiles('machinedata.json')
 452         Utilities.addArchival(newJob, archiveSettings)
 453
 454         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 455
 456         // For perf, we need to keep the run results longer
 457         newJob.with {
 458             // Enable the log rotator
 459             logRotator {
 460                 artifactDaysToKeep(7)
 461                 daysToKeep(300)
 462                 artifactNumToKeep(25)
 463                 numToKeep(1000)
 464             }
 465         }
 466     } // os
 467
 468     def flowJobTPRunList = throughputOSList.collect { os ->
 469         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
 470     }
 471     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
 472         if (isPR) {
 473             parameters {
 474                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 475             }
 476         }
 477         buildFlow("""
 478 // First, build the bits on RHEL7.2
 479 b = build(params, '${fullBuildJobName}')
 480
 481 // Then, run the perf tests
 482 parallel(
 483     ${flowJobTPRunList.join(",\n    ")}
 484 )
 485 """)
 486     }
 487
 488     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
 489     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
 490
 491     if (isPR) {
 492         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 493         builder.setGithubContext("Linux Throughput Perf Test Flow")
 494         builder.triggerOnlyOnComment()
 495         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
 496         builder.triggerForBranch(branch)
 497         builder.emitTrigger(newFlowJob)
 498     }
 499     else {
 500         // Set a push trigger
 501         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 502         builder.emitTrigger(newFlowJob)
 503     }
 504
 505 } // isPR
 506
 507 // Setup ILLink tests
 508 [true, false].each { isPR ->
 509     ['Windows_NT'].each { os ->
 510         ['x64'].each { arch ->
 511             def architecture = arch
 512             def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}", isPR)) {
 513                 // Set the label.
 514                 label('windows_clr_perf')
 515                 wrappers {
 516                     credentialsBinding {
 517                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 518                     }
 519                 }
 520
 521                 if (isPR)
 522                 {
 523                     parameters
 524                     {
 525                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 526                     }
 527                 }
 528
 529                 parameters
 530                 {
 531                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
 532                     stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
 533                 }
 534                 def configuration = 'Release'
 535                 def runType = isPR ? 'private' : 'rolling'
 536                 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 537                 def uploadString = '-uploadToBenchview'
 538
 539                 steps {
 540                     // Batch
 541                     batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
 542                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 543                     batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 544
 545                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 546                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 547                     batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
 548                     "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
 549                     "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
 550                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
 551                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 552                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 553                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 554
 555                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 556
 557                     // Run with just stopwatch: Profile=Off
 558                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -nowarmup -runtype ${runType} -scenarioTest -group ILLink")
 559                 }
 560              }
 561
 562              // Save machinedata.json to /artifact/bin/ Jenkins dir
 563             def archiveSettings = new ArchivalSettings()
 564             archiveSettings.addFiles('Perf-*.xml')
 565             archiveSettings.addFiles('Perf-*.log')
 566             archiveSettings.addFiles('machinedata.json')
 567             Utilities.addArchival(newJob, archiveSettings)
 568
 569             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 570
 571             newJob.with {
 572                 wrappers {
 573                     timeout {
 574                         absolute(240)
 575                     }
 576                 }
 577             }
 578
 579             if (isPR) {
 580                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 581                 builder.setGithubContext("${os} ${arch} ILLink Perf Tests")
 582                 builder.triggerOnlyOnComment()
 583                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink\\W+perf.*")
 584                 builder.triggerForBranch(branch)
 585                 builder.emitTrigger(newJob)
 586             }
 587             else {
 588                 // Set a push trigger
 589                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 590                 builder.emitTrigger(newJob)
 591             }
 592         }
 593     }
 594 }
 595
 596 Utilities.createHelperJob(this, project, branch,
 597     "Welcome to the ${project} Perf help",
 598     "Have a nice day!")