1 // Import the utility functionality.
3 import jobs.generation.*;
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
10 def static getOSGroup(def os) {
11 def osGroupMap = ['Ubuntu14.04':'Linux',
13 'Ubuntu16.04': 'Linux',
17 'Windows_NT':'Windows_NT',
20 'OpenSUSE13.2': 'Linux',
21 'OpenSUSE42.1': 'Linux',
22 'LinuxARMEmulator': 'Linux']
23 def osGroup = osGroupMap.get(os, null)
24 assert osGroup != null : "Could not find os group for ${os}"
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30 ['Windows_NT'].each { os ->
31 ['x64', 'x86'].each { arch ->
32 [true, false].each { isSmoketest ->
33 ['ryujit'].each { jit ->
34 ['full_opt', 'min_opt'].each { opt_level ->
36 def architecture = arch
37 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
39 def python = "C:\\Python35\\python.exe"
41 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
44 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
45 python = "C:\\python3.7.0\\python.exe"
48 label('windows_server_2016_clr_perf')
53 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
59 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
65 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
66 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
71 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
72 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
76 def configuration = 'Release'
77 def runType = isPR ? 'private' : 'rolling'
78 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
79 def uploadString = isSmoketest ? '' : '-uploadToBenchview'
84 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
85 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
86 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
87 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
88 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
89 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
90 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
91 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
92 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
93 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
94 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
95 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
97 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
99 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
101 // Run with just stopwatch: Profile=Off
102 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
103 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
105 // Run with the full set of counters enabled: Profile=On
106 if (opt_level != 'min_opt') {
107 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
108 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
113 def archiveSettings = new ArchivalSettings()
114 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
115 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
116 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
117 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
118 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
119 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
120 archiveSettings.addFiles('machinedata.json')
121 archiveSettings.setAlwaysArchive()
123 Utilities.addArchival(newJob, archiveSettings)
124 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
128 artifactDaysToKeep(14)
130 artifactNumToKeep(100)
141 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
143 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
146 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
149 if (opt_level == 'min_opt') {
150 opts = '\\W+min_opts'
153 if (jit != 'ryujit') {
157 builder.triggerOnlyOnComment()
158 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
161 builder.triggerForBranch(branch)
162 builder.emitTrigger(newJob)
164 else if (opt_level == 'full_opt') {
165 // Set a push trigger
166 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
167 builder.emitTrigger(newJob)
170 // Set periodic trigger
171 Utilities.addPeriodicTrigger(newJob, '@daily')
180 // Setup throughput perflab tests runs
181 [true, false].each { isPR ->
182 ['Windows_NT'].each { os ->
183 ['x64', 'x86'].each { arch ->
184 ['ryujit'].each { jit ->
185 [true, false].each { pgo_optimized ->
186 ['full_opt', 'min_opt'].each { opt_level ->
187 def architecture = arch
189 def python = "C:\\Python35\\python.exe"
194 if (!pgo_optimized) {
195 pgo_build = " -nopgooptimize"
200 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
202 label('windows_server_2016_clr_perf')
205 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
211 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
215 def configuration = 'Release'
216 def runType = isPR ? 'private' : 'rolling'
217 def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
221 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
222 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
223 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
224 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
225 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
226 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
227 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
228 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
229 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
230 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
231 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
232 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
233 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
234 batchFile("${python} -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
238 // Save machinedata.json to /artifact/bin/ Jenkins dir
239 def archiveSettings = new ArchivalSettings()
240 archiveSettings.addFiles('throughput-*.csv')
241 archiveSettings.setAlwaysArchive()
242 Utilities.addArchival(newJob, archiveSettings)
244 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
248 if (opt_level == 'min_opt') {
249 opts = '\\W+min_opts'
253 if (jit != 'ryujit') {
259 pgo_trigger = "\\W+nopgo"
263 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
264 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
265 builder.triggerOnlyOnComment()
266 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
267 builder.triggerForBranch(branch)
268 builder.emitTrigger(newJob)
270 else if (opt_level == 'full_opt' && pgo_optimized) {
271 // Set a push trigger
272 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
273 builder.emitTrigger(newJob)
276 // Set periodic trigger
277 Utilities.addPeriodicTrigger(newJob, '@daily')
286 def static getFullPerfJobName(def project, def os, def arch, def isPR) {
287 return Utilities.getFullJobName(project, "perf_${os}_${arch}", isPR)
290 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
291 [true, false].each { isPR ->
292 ['x64'].each { architecture ->
293 def fullBuildJobName = Utilities.getFullJobName(project, "perf_linux_build", isPR)
294 def configuration = 'Release'
296 def crossCompile = ""
298 def python = "python3.5"
300 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
301 ['RHEL7.2'].each { os ->
302 def newBuildJob = job(fullBuildJobName) {
304 shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
305 shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}${crossLayout}")
308 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
309 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
310 Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
314 // Actual perf testing on the following OSes
315 def perfOSList = ['Ubuntu16.04']
317 perfOSList.each { os ->
318 def newJob = job(getFullPerfJobName(project, os, architecture, isPR)) {
320 def machineLabel = 'ubuntu_1604_clr_perf'
325 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
331 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
336 // Cap the maximum number of iterations to 21.
337 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
338 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
339 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
342 def osGroup = getOSGroup(os)
343 def runType = isPR ? 'private' : 'rolling'
344 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
345 def uploadString = '-uploadToBenchview'
347 def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
350 shell("./tests/scripts/perf-prep.sh --nocorefx")
351 shell("./init-tools.sh")
352 copyArtifacts(fullBuildJobName) {
353 includePatterns("bin/**")
355 buildNumber('\${PRODUCT_BUILD}')
358 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
359 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
360 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
361 shell("""${python} ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
365 def archiveSettings = new ArchivalSettings()
366 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
367 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
368 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
369 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
370 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
371 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
372 archiveSettings.addFiles('machinedata.json')
373 archiveSettings.setAlwaysArchive()
375 Utilities.addArchival(newJob, archiveSettings)
376 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
378 // For perf, we need to keep the run results longer
380 // Enable the log rotator
382 artifactDaysToKeep(14)
384 artifactNumToKeep(100)
395 def flowJobPerfRunList = perfOSList.collect { os ->
396 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, architecture, isPR)}') }"
398 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_${architecture}_flow", isPR, '')) {
401 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
405 // First, build the bits on RHEL7.2
406 b = build(params, '${fullBuildJobName}')
408 // Then, run the perf tests
410 ${flowJobPerfRunList.join(",\n ")}
415 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
416 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
419 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
420 builder.setGithubContext("Linux Perf Test Flow")
421 builder.triggerOnlyOnComment()
422 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
423 builder.triggerForBranch(branch)
424 builder.emitTrigger(newFlowJob)
427 // Set a push trigger
428 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
429 builder.emitTrigger(newFlowJob)
434 def static getDockerImageName(def architecture, def os, def isBuild) {
435 // We must change some docker private images to official later
437 if (architecture == 'arm') {
438 if (os == 'Ubuntu') {
439 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
443 println("Unknown architecture to use docker: ${architecture} ${os}");
447 def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
448 return Utilities.getFullJobName(project, "perf_throughput_${os}_${arch}", isPR)
451 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
452 [true, false].each { isPR ->
453 ['x64','arm'].each { architecture ->
454 def fullBuildJobName = Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_build", isPR)
455 def configuration = 'Release'
458 def crossCompile = ""
459 def python = "python3.5"
461 if (architecture == "arm") {
463 def buildCommands = []
464 def newBuildJob = job(fullBuildJobName) {
465 def additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
466 def dockerImage = getDockerImageName(architecture, 'Ubuntu', true)
467 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
469 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh release ${architecture} cross"
472 buildCommands.each { buildCommand ->
478 azureVMAgentPostBuildAction {
479 agentPostBuildAction('Delete agent after build execution (when idle).')
483 Utilities.setMachineAffinity(newBuildJob, "Ubuntu16.04", 'latest-or-auto')
484 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
485 Utilities.addArchival(newBuildJob, "bin/Product/**")
488 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
489 ['RHEL7.2'].each { os ->
490 def newBuildJob = job(fullBuildJobName) {
492 shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
495 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
496 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
497 Utilities.addArchival(newBuildJob, "bin/Product/**")
501 // Actual perf testing on the following OSes
502 def throughputOSList = ['Ubuntu16.04']
503 if (architecture == 'arm') {
504 throughputOSList = ['Ubuntu14.04']
506 def throughputOptLevelList = ['full_opt', 'min_opt']
508 def throughputOSOptLevelList = []
510 throughputOSList.each { os ->
511 throughputOptLevelList.each { opt_level ->
512 throughputOSOptLevelList.add("${os}_${opt_level}")
516 throughputOSList.each { os ->
517 throughputOptLevelList.each { opt_level ->
518 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", architecture, isPR)) {
520 def machineLabel = 'ubuntu_1604_clr_perf'
521 if (architecture == 'arm') {
522 machineLabel = 'ubuntu_1404_clr_perf_arm'
528 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
534 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
539 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
542 def osGroup = getOSGroup(os)
543 def runType = isPR ? 'private' : 'rolling'
544 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
545 def archString = architecture == 'arm' ? ' --arch=arm' : ''
548 shell("bash ./tests/scripts/perf-prep.sh --throughput${archString}")
549 copyArtifacts(fullBuildJobName) {
550 includePatterns("bin/Product/**")
552 buildNumber('\${PRODUCT_BUILD}')
555 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
556 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
557 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
558 shell("""${python} ./tests/scripts/run-throughput-perf.py \\
559 -arch \"${architecture}\" \\
561 -configuration \"${configuration}\" \\
562 -opt_level \"${opt_level}\" \\
563 -clr_root \"\${WORKSPACE}\" \\
564 -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/lib\" \\
565 -run_type \"${runType}\" \\
566 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
570 // Save machinedata.json to /artifact/bin/ Jenkins dir
571 def archiveSettings = new ArchivalSettings()
572 archiveSettings.addFiles('throughput-*.csv')
573 archiveSettings.addFiles('machinedata.json')
574 archiveSettings.setAlwaysArchive()
575 Utilities.addArchival(newJob, archiveSettings)
577 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
579 // For perf, we need to keep the run results longer
581 // Enable the log rotator
583 artifactDaysToKeep(7)
585 artifactNumToKeep(25)
592 def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
593 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, architecture, isPR)}') }"
595 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_flow", isPR, '')) {
598 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
602 // First, build the bits on RHEL7.2
603 b = build(params, '${fullBuildJobName}')
605 // Then, run the perf tests
607 ${flowJobTPRunList.join(",\n ")}
612 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
613 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
616 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
617 builder.setGithubContext("Linux ${architecture} Throughput Perf Test Flow")
618 builder.triggerOnlyOnComment()
619 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
620 builder.triggerForBranch(branch)
621 builder.emitTrigger(newFlowJob)
624 // Set a push trigger
625 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
626 builder.emitTrigger(newFlowJob)
631 // Setup CoreCLR-Scenarios tests
632 [true, false].each { isPR ->
633 ['Windows_NT'].each { os ->
634 ['x64', 'x86'].each { arch ->
635 ['ryujit'].each { jit ->
636 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
637 def architecture = arch
638 def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
641 def python = "C:\\Python35\\python.exe"
644 label('windows_server_2016_clr_perf')
647 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
653 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
658 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
659 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
662 def configuration = 'Release'
663 def runType = isPR ? 'private' : 'rolling'
664 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
665 def uploadString = '-uploadToBenchview'
669 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
670 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
671 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
673 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
674 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
675 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
676 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
677 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
678 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
679 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
680 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
681 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
683 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
685 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
688 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
691 if (opt_level != 'min_opt') {
692 batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
697 def archiveSettings = new ArchivalSettings()
698 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
699 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
700 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
701 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
702 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
703 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
704 archiveSettings.addFiles('machinedata.json')
705 archiveSettings.setAlwaysArchive()
707 Utilities.addArchival(newJob, archiveSettings)
708 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
712 artifactDaysToKeep(14)
714 artifactNumToKeep(100)
726 if (opt_level == 'min_opt') {
727 opts = '\\W+min_opts'
730 if (jit != 'ryujit') {
734 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
735 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
736 builder.triggerOnlyOnComment()
737 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
738 builder.triggerForBranch(branch)
739 builder.emitTrigger(newJob)
741 else if (opt_level == 'full_opt') {
742 // Set a push trigger
743 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
744 builder.emitTrigger(newJob)
747 // Set periodic trigger
748 Utilities.addPeriodicTrigger(newJob, '@daily')
756 // Setup size-on-disk test
757 ['Windows_NT'].each { os ->
758 ['x64', 'x86'].each { arch ->
759 def architecture = arch
760 def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
761 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
765 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
769 def channel = 'master'
770 def configuration = 'Release'
771 def runType = 'rolling'
772 def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
773 def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
774 def coreRoot = "${testBin}\\Tests\\Core_Root"
775 def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
776 def python = "C:\\python3.7.0\\python.exe"
779 // Install nuget and get BenchView tools
780 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
781 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
782 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
784 // Generate submission metadata for BenchView
785 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
786 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
787 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
788 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
789 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
790 "${python} \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
791 "${python} \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
793 // Generate machine data from BenchView
794 batchFile("${python} \"${benchViewTools}\\machinedata.py\"")
796 // Build CoreCLR and gnerate test layout
797 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
798 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
800 // Run the size on disk benchmark
801 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
803 // From sodbench.csv, create measurment.json, then submission.json
804 batchFile("${python} \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
805 batchFile("${python} \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
807 // If this is a PR, upload submission.json
808 batchFile("${python} \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
812 def archiveSettings = new ArchivalSettings()
813 archiveSettings.addFiles('bin/toArchive/**')
814 archiveSettings.addFiles('machinedata.json')
815 archiveSettings.setAlwaysArchive()
817 Utilities.addArchival(newJob, archiveSettings)
818 Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
820 // Set the cron job here. We run nightly on each flavor, regardless of code changes
821 Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
825 artifactDaysToKeep(14)
827 artifactNumToKeep(100)
839 // Setup IlLink tests
840 [true, false].each { isPR ->
841 ['Windows_NT'].each { os ->
842 ['x64'].each { arch ->
843 ['ryujit'].each { jit ->
844 ['full_opt'].each { opt_level ->
845 def architecture = arch
846 def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
847 label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
850 def python = "C:\\python3.7.0\\python.exe"
853 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
859 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
864 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
865 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
868 def configuration = 'Release'
869 def runType = isPR ? 'private' : 'rolling'
870 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
871 def uploadString = '-uploadToBenchview'
875 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
876 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
877 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
879 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
880 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
881 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
882 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
883 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
884 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
885 "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
886 batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
887 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
889 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
891 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
894 batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
895 "${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
899 def archiveSettings = new ArchivalSettings()
900 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
901 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
902 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
903 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
904 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
905 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
906 archiveSettings.addFiles('machinedata.json')
907 archiveSettings.setAlwaysArchive()
909 // Set the label (currently we are only measuring size, therefore we are running on VM).
910 Utilities.addArchival(newJob, archiveSettings)
911 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
915 artifactDaysToKeep(14)
917 artifactNumToKeep(100)
928 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
929 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
930 builder.triggerOnlyOnComment()
931 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
932 builder.triggerForBranch(branch)
933 builder.emitTrigger(newJob)
936 // Set a push trigger
937 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
938 builder.emitTrigger(newJob)
946 Utilities.createHelperJob(this, project, branch,
947 "Welcome to the ${project} Perf help",