1 // Import the utility functionality.
3 import jobs.generation.*;
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
10 def static getOSGroup(def os) {
11 def osGroupMap = ['Ubuntu14.04':'Linux',
13 'Ubuntu16.04': 'Linux',
17 'Windows_NT':'Windows_NT',
20 'OpenSUSE13.2': 'Linux',
21 'OpenSUSE42.1': 'Linux',
22 'LinuxARMEmulator': 'Linux']
23 def osGroup = osGroupMap.get(os, null)
24 assert osGroup != null : "Could not find os group for ${os}"
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30 ['Windows_NT'].each { os ->
31 ['x64', 'x86'].each { arch ->
32 [true, false].each { isSmoketest ->
33 ['ryujit'].each { jit ->
34 ['full_opt', 'min_opt'].each { opt_level ->
36 def architecture = arch
37 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
39 def python = "C:\\Python35\\python.exe"
41 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
44 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
45 python = "C:\\python3.7.0\\python.exe"
48 label('windows_server_2016_clr_perf')
53 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
59 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
65 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
66 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
71 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
72 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
76 def configuration = 'Release'
77 def runType = isPR ? 'private' : 'rolling'
78 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
79 def uploadString = isSmoketest ? '' : '-uploadToBenchview'
84 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
85 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
86 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
87 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
88 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
89 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
90 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
91 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
92 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
93 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
94 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
95 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
97 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
99 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
101 // Run with just stopwatch: Profile=Off
102 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
103 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
105 // Run with the full set of counters enabled: Profile=On
106 if (opt_level != 'min_opt') {
107 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
108 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
113 def archiveSettings = new ArchivalSettings()
114 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
115 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
116 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
117 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
118 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
119 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
120 archiveSettings.addFiles('machinedata.json')
121 archiveSettings.setAlwaysArchive()
123 Utilities.addArchival(newJob, archiveSettings)
124 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
128 artifactDaysToKeep(14)
130 artifactNumToKeep(100)
141 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
143 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
146 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
149 if (opt_level == 'min_opt') {
150 opts = '\\W+min_opts'
153 if (jit != 'ryujit') {
157 builder.triggerOnlyOnComment()
158 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
161 builder.triggerForBranch(branch)
162 builder.emitTrigger(newJob)
164 else if (opt_level == 'full_opt') {
165 // Set a push trigger
166 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
167 builder.emitTrigger(newJob)
170 // Set periodic trigger
171 Utilities.addPeriodicTrigger(newJob, '@daily')
180 // Setup throughput perflab tests runs
181 [true, false].each { isPR ->
182 ['Windows_NT'].each { os ->
183 ['x64', 'x86'].each { arch ->
184 ['ryujit'].each { jit ->
185 [true, false].each { pgo_optimized ->
186 ['full_opt', 'min_opt'].each { opt_level ->
187 def architecture = arch
189 def python = "C:\\Python35\\python.exe"
194 if (!pgo_optimized) {
195 pgo_build = " -nopgooptimize"
200 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
202 label('windows_server_2016_clr_perf')
205 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
211 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
215 def configuration = 'Release'
216 def runType = isPR ? 'private' : 'rolling'
217 def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
221 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
222 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
223 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
224 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
225 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
226 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
227 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
228 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
229 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
230 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
231 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
232 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
233 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
234 batchFile("${python} -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
238 // Save machinedata.json to /artifact/bin/ Jenkins dir
239 def archiveSettings = new ArchivalSettings()
240 archiveSettings.addFiles('throughput-*.csv')
241 archiveSettings.setAlwaysArchive()
242 Utilities.addArchival(newJob, archiveSettings)
244 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
248 if (opt_level == 'min_opt') {
249 opts = '\\W+min_opts'
253 if (jit != 'ryujit') {
259 pgo_trigger = "\\W+nopgo"
263 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
264 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
265 builder.triggerOnlyOnComment()
266 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
267 builder.triggerForBranch(branch)
268 builder.emitTrigger(newJob)
270 else if (opt_level == 'full_opt' && pgo_optimized) {
271 // Set a push trigger
272 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
273 builder.emitTrigger(newJob)
276 // Set periodic trigger
277 Utilities.addPeriodicTrigger(newJob, '@daily')
286 def static getFullPerfJobName(def project, def os, def arch, def isPR) {
287 return Utilities.getFullJobName(project, "perf_${os}_${arch}", isPR)
290 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
291 [true, false].each { isPR ->
292 ['x64'].each { architecture ->
293 def fullBuildJobName = Utilities.getFullJobName(project, "perf_linux_build", isPR)
294 def configuration = 'Release'
296 def crossCompile = ""
298 def python = "python3.5"
300 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
301 ['RHEL7.2'].each { os ->
302 def newBuildJob = job(fullBuildJobName) {
304 shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
305 shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}${crossLayout}")
308 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
309 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
310 Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
314 // Actual perf testing on the following OSes
315 def perfOSList = ['Ubuntu16.04']
317 perfOSList.each { os ->
318 def newJob = job(getFullPerfJobName(project, os, architecture, isPR)) {
320 def machineLabel = 'ubuntu_1604_clr_perf'
325 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
331 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
336 // Cap the maximum number of iterations to 21.
337 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
338 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
339 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
342 def osGroup = getOSGroup(os)
343 def runType = isPR ? 'private' : 'rolling'
344 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
345 def uploadString = '-uploadToBenchview'
347 def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
350 shell("./tests/scripts/perf-prep.sh --nocorefx")
351 shell("./init-tools.sh")
352 copyArtifacts(fullBuildJobName) {
353 includePatterns("bin/**")
355 buildNumber('\${PRODUCT_BUILD}')
358 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
359 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
360 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
361 shell("""${python} ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
365 def archiveSettings = new ArchivalSettings()
366 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
367 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
368 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
369 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
370 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
371 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
372 archiveSettings.addFiles('machinedata.json')
373 archiveSettings.setAlwaysArchive()
375 Utilities.addArchival(newJob, archiveSettings)
376 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
378 // For perf, we need to keep the run results longer
380 // Enable the log rotator
382 artifactDaysToKeep(14)
384 artifactNumToKeep(100)
395 def flowJobPerfRunList = perfOSList.collect { os ->
396 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, architecture, isPR)}') }"
398 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_${architecture}_flow", isPR, '')) {
401 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
405 // First, build the bits on RHEL7.2
406 b = build(params, '${fullBuildJobName}')
408 // Then, run the perf tests
410 ${flowJobPerfRunList.join(",\n ")}
415 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
416 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
419 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
420 builder.setGithubContext("Linux Perf Test Flow")
421 builder.triggerOnlyOnComment()
422 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
423 builder.triggerForBranch(branch)
424 builder.emitTrigger(newFlowJob)
427 // Set a push trigger
428 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
429 builder.emitTrigger(newFlowJob)
434 def static getDockerImageName(def architecture, def os, def isBuild) {
435 // We must change some docker private images to official later
437 if (architecture == 'arm') {
438 if (os == 'Ubuntu') {
439 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
443 println("Unknown architecture to use docker: ${architecture} ${os}");
447 def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
448 return Utilities.getFullJobName(project, "perf_throughput_${os}_${arch}", isPR)
451 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
452 [true, false].each { isPR ->
453 ['x64','arm'].each { architecture ->
454 def fullBuildJobName = Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_build", isPR)
455 def configuration = 'Release'
458 def crossCompile = ""
459 def python = "python3.5"
460 def tgzFile = "bin-Product-Linux.${architecture}.${configuration}.tgz"
462 if (architecture == "arm") {
464 def buildCommands = []
465 def newBuildJob = job(fullBuildJobName) {
466 def additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
467 def dockerImage = getDockerImageName(architecture, 'Ubuntu', true)
468 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
470 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh release ${architecture} cross"
473 buildCommands.each { buildCommand ->
475 shell("tar -czf ${tgzFile} bin/Product/Linux.${architecture}.${configuration}")
480 azureVMAgentPostBuildAction {
481 agentPostBuildAction('Delete agent after build execution (when idle).')
485 Utilities.setMachineAffinity(newBuildJob, "Ubuntu16.04", 'latest-or-auto')
486 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
487 Utilities.addArchival(newBuildJob, "${tgzFile}")
490 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
491 ['RHEL7.2'].each { os ->
492 def newBuildJob = job(fullBuildJobName) {
494 shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
497 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
498 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
499 Utilities.addArchival(newBuildJob, "bin/Product/**")
503 // Actual perf testing on the following OSes
504 def throughputOSList = ['Ubuntu16.04']
505 if (architecture == 'arm') {
506 throughputOSList = ['Ubuntu14.04']
508 def throughputOptLevelList = ['full_opt', 'min_opt']
510 def throughputOSOptLevelList = []
512 throughputOSList.each { os ->
513 throughputOptLevelList.each { opt_level ->
514 throughputOSOptLevelList.add("${os}_${opt_level}")
518 throughputOSList.each { os ->
519 throughputOptLevelList.each { opt_level ->
520 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", architecture, isPR)) {
522 def machineLabel = 'ubuntu_1604_clr_perf'
523 if (architecture == 'arm') {
524 machineLabel = 'ubuntu_1404_clr_perf_arm'
530 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
536 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
541 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
544 def osGroup = getOSGroup(os)
545 def runType = isPR ? 'private' : 'rolling'
546 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
547 def archString = architecture == 'arm' ? ' --arch=arm' : ''
550 shell("bash ./tests/scripts/perf-prep.sh --throughput${archString}")
551 copyArtifacts(fullBuildJobName) {
552 if (architecture == 'arm') {
553 includePatterns("${tgzFile}")
556 includePatterns("bin/Product/**")
559 buildNumber('\${PRODUCT_BUILD}')
562 if (architecture == 'arm') {
563 shell("tar -xzf ./${tgzFile} || exit 0")
565 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
566 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
567 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
568 shell("""${python} ./tests/scripts/run-throughput-perf.py \\
569 -arch \"${architecture}\" \\
571 -configuration \"${configuration}\" \\
572 -opt_level \"${opt_level}\" \\
573 -clr_root \"\${WORKSPACE}\" \\
574 -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/lib\" \\
575 -run_type \"${runType}\" \\
576 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
580 // Save machinedata.json to /artifact/bin/ Jenkins dir
581 def archiveSettings = new ArchivalSettings()
582 archiveSettings.addFiles('throughput-*.csv')
583 archiveSettings.addFiles('machinedata.json')
584 archiveSettings.setAlwaysArchive()
585 Utilities.addArchival(newJob, archiveSettings)
587 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
589 // For perf, we need to keep the run results longer
591 // Enable the log rotator
593 artifactDaysToKeep(7)
595 artifactNumToKeep(25)
602 def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
603 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, architecture, isPR)}') }"
605 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_flow", isPR, '')) {
608 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
612 // First, build the bits on RHEL7.2
613 b = build(params, '${fullBuildJobName}')
615 // Then, run the perf tests
617 ${flowJobTPRunList.join(",\n ")}
622 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
623 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
626 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
627 builder.setGithubContext("Linux ${architecture} Throughput Perf Test Flow")
628 builder.triggerOnlyOnComment()
629 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
630 builder.triggerForBranch(branch)
631 builder.emitTrigger(newFlowJob)
634 // Set a push trigger
635 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
636 builder.emitTrigger(newFlowJob)
641 // Setup CoreCLR-Scenarios tests
642 [true, false].each { isPR ->
643 ['Windows_NT'].each { os ->
644 ['x64', 'x86'].each { arch ->
645 ['ryujit'].each { jit ->
646 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
647 def architecture = arch
648 def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
651 def python = "C:\\Python35\\python.exe"
654 label('windows_server_2016_clr_perf')
657 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
663 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
668 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
669 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
672 def configuration = 'Release'
673 def runType = isPR ? 'private' : 'rolling'
674 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
675 def uploadString = '-uploadToBenchview'
679 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
680 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
681 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
683 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
684 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
685 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
686 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
687 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
688 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
689 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
690 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
691 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
693 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
695 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
698 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
701 if (opt_level != 'min_opt') {
702 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
707 def archiveSettings = new ArchivalSettings()
708 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
709 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
710 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
711 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
712 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
713 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
714 archiveSettings.addFiles('machinedata.json')
715 archiveSettings.setAlwaysArchive()
717 Utilities.addArchival(newJob, archiveSettings)
718 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
722 artifactDaysToKeep(14)
724 artifactNumToKeep(100)
736 if (opt_level == 'min_opt') {
737 opts = '\\W+min_opts'
740 if (jit != 'ryujit') {
744 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
745 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
746 builder.triggerOnlyOnComment()
747 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
748 builder.triggerForBranch(branch)
749 builder.emitTrigger(newJob)
751 else if (opt_level == 'full_opt') {
752 // Set a push trigger
753 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
754 builder.emitTrigger(newJob)
757 // Set periodic trigger
758 Utilities.addPeriodicTrigger(newJob, '@daily')
766 // Setup size-on-disk test
767 ['Windows_NT'].each { os ->
768 ['x64', 'x86'].each { arch ->
769 def architecture = arch
770 def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
771 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
775 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
779 def channel = 'master'
780 def configuration = 'Release'
781 def runType = 'rolling'
782 def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
783 def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
784 def coreRoot = "${testBin}\\Tests\\Core_Root"
785 def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
786 def python = "C:\\python3.7.0\\python.exe"
789 // Install nuget and get BenchView tools
790 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
791 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
792 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
794 // Generate submission metadata for BenchView
795 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
796 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
797 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
798 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
799 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
800 "${python} \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
801 "${python} \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
803 // Generate machine data from BenchView
804 batchFile("${python} \"${benchViewTools}\\machinedata.py\"")
806 // Build CoreCLR and gnerate test layout
807 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
808 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
810 // Run the size on disk benchmark
811 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
813 // From sodbench.csv, create measurment.json, then submission.json
814 batchFile("${python} \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
815 batchFile("${python} \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
817 // If this is a PR, upload submission.json
818 batchFile("${python} \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
822 def archiveSettings = new ArchivalSettings()
823 archiveSettings.addFiles('bin/toArchive/**')
824 archiveSettings.addFiles('machinedata.json')
825 archiveSettings.setAlwaysArchive()
827 Utilities.addArchival(newJob, archiveSettings)
828 Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
830 // Set the cron job here. We run nightly on each flavor, regardless of code changes
831 Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
835 artifactDaysToKeep(14)
837 artifactNumToKeep(100)
849 // Setup IlLink tests
850 [true, false].each { isPR ->
851 ['Windows_NT'].each { os ->
852 ['x64'].each { arch ->
853 ['ryujit'].each { jit ->
854 ['full_opt'].each { opt_level ->
855 def architecture = arch
856 def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
857 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
860 def python = "C:\\python3.7.0\\python.exe"
863 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
869 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
874 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
875 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
878 def configuration = 'Release'
879 def runType = isPR ? 'private' : 'rolling'
880 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
881 def uploadString = '-uploadToBenchview'
885 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
886 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
887 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
889 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
890 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
891 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
892 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
893 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
894 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
895 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
896 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
897 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
899 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
901 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
904 batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
905 "${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
909 def archiveSettings = new ArchivalSettings()
910 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
911 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
912 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
913 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
914 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
915 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
916 archiveSettings.addFiles('machinedata.json')
917 archiveSettings.setAlwaysArchive()
919 // Set the label (currently we are only measuring size, therefore we are running on VM).
920 Utilities.addArchival(newJob, archiveSettings)
921 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
925 artifactDaysToKeep(14)
927 artifactNumToKeep(100)
938 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
939 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
940 builder.triggerOnlyOnComment()
941 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
942 builder.triggerForBranch(branch)
943 builder.emitTrigger(newJob)
946 // Set a push trigger
947 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
948 builder.emitTrigger(newJob)
956 Utilities.createHelperJob(this, project, branch,
957 "Welcome to the ${project} Perf help",