1 // Import the utility functionality.
3 import jobs.generation.*;
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
10 def static getOSGroup(def os) {
11 def osGroupMap = ['Ubuntu14.04':'Linux',
13 'Ubuntu16.04': 'Linux',
17 'Windows_NT':'Windows_NT',
20 'OpenSUSE13.2': 'Linux',
21 'OpenSUSE42.1': 'Linux',
22 'LinuxARMEmulator': 'Linux']
23 def osGroup = osGroupMap.get(os, null)
24 assert osGroup != null : "Could not find os group for ${os}"
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30 ['Windows_NT'].each { os ->
31 ['x64', 'x86'].each { arch ->
32 [true, false].each { isSmoketest ->
33 ['ryujit'].each { jit ->
34 ['full_opt', 'min_opt'].each { opt_level ->
36 def architecture = arch
37 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
40 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
42 label('windows_server_2016_clr_perf')
45 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
51 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
57 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
58 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
63 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
64 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
68 def configuration = 'Release'
69 def runType = isPR ? 'private' : 'rolling'
70 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
71 def uploadString = isSmoketest ? '' : '-uploadToBenchview'
76 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
77 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
78 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
79 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
80 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
81 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
82 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
83 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
84 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
85 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
86 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
87 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
89 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
91 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
93 // Run with just stopwatch: Profile=Off
94 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
95 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
97 // Run with the full set of counters enabled: Profile=On
98 if (opt_level != 'min_opt') {
99 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
100 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
106 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
108 def archiveSettings = new ArchivalSettings()
109 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
110 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
111 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
112 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
113 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
114 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
115 archiveSettings.addFiles('machinedata.json')
116 archiveSettings.setAlwaysArchive()
118 Utilities.addArchival(newJob, archiveSettings)
119 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
123 artifactDaysToKeep(14)
125 artifactNumToKeep(100)
136 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
138 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
141 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
144 if (opt_level == 'min_opt') {
145 opts = '\\W+min_opts'
148 if (jit != 'ryujit') {
152 builder.triggerOnlyOnComment()
153 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
156 builder.triggerForBranch(branch)
157 builder.emitTrigger(newJob)
159 else if (opt_level == 'full_opt') {
160 // Set a push trigger
161 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
162 builder.emitTrigger(newJob)
165 // Set periodic trigger
166 Utilities.addPeriodicTrigger(newJob, '@daily')
175 // Setup throughput perflab tests runs
176 [true, false].each { isPR ->
177 ['Windows_NT'].each { os ->
178 ['x64', 'x86'].each { arch ->
179 ['ryujit'].each { jit ->
180 [true, false].each { pgo_optimized ->
181 ['full_opt', 'min_opt'].each { opt_level ->
182 def architecture = arch
187 if (!pgo_optimized) {
188 pgo_build = " -nopgooptimize"
193 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
195 label('windows_server_2016_clr_perf')
198 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
204 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
208 def configuration = 'Release'
209 def runType = isPR ? 'private' : 'rolling'
210 def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
214 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
215 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
216 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
217 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
218 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
219 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
220 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
221 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
222 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
223 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
224 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
225 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
226 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
227 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
231 // Save machinedata.json to /artifact/bin/ Jenkins dir
232 def archiveSettings = new ArchivalSettings()
233 archiveSettings.addFiles('throughput-*.csv')
234 archiveSettings.setAlwaysArchive()
235 Utilities.addArchival(newJob, archiveSettings)
237 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
241 if (opt_level == 'min_opt') {
242 opts = '\\W+min_opts'
246 if (jit != 'ryujit') {
252 pgo_trigger = "\\W+nopgo"
256 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
257 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
258 builder.triggerOnlyOnComment()
259 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
260 builder.triggerForBranch(branch)
261 builder.emitTrigger(newJob)
263 else if (opt_level == 'full_opt' && pgo_optimized) {
264 // Set a push trigger
265 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
266 builder.emitTrigger(newJob)
269 // Set periodic trigger
270 Utilities.addPeriodicTrigger(newJob, '@daily')
279 def static getFullPerfJobName(def project, def os, def arch, def isPR) {
280 return Utilities.getFullJobName(project, "perf_${os}_${arch}", isPR)
283 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
284 [true, false].each { isPR ->
285 ['x64'].each { architecture ->
286 def fullBuildJobName = Utilities.getFullJobName(project, "perf_linux_build", isPR)
287 def configuration = 'Release'
289 def crossCompile = ""
291 def python = "python3.5"
293 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
294 ['RHEL7.2'].each { os ->
295 def newBuildJob = job(fullBuildJobName) {
297 shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
298 shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}${crossLayout}")
301 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
302 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
303 Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
307 // Actual perf testing on the following OSes
308 def perfOSList = ['Ubuntu16.04']
310 perfOSList.each { os ->
311 def newJob = job(getFullPerfJobName(project, os, architecture, isPR)) {
313 def machineLabel = 'ubuntu_1604_clr_perf'
318 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
324 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
329 // Cap the maximum number of iterations to 21.
330 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
331 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
332 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
335 def osGroup = getOSGroup(os)
336 def runType = isPR ? 'private' : 'rolling'
337 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
338 def uploadString = '-uploadToBenchview'
340 def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
343 shell("./tests/scripts/perf-prep.sh --nocorefx")
344 shell("./init-tools.sh")
345 copyArtifacts(fullBuildJobName) {
346 includePatterns("bin/**")
348 buildNumber('\${PRODUCT_BUILD}')
351 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
352 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
353 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
354 shell("""${python} ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
358 def archiveSettings = new ArchivalSettings()
359 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
360 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
361 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
362 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
363 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
364 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
365 archiveSettings.addFiles('machinedata.json')
366 archiveSettings.setAlwaysArchive()
368 Utilities.addArchival(newJob, archiveSettings)
369 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
371 // For perf, we need to keep the run results longer
373 // Enable the log rotator
375 artifactDaysToKeep(14)
377 artifactNumToKeep(100)
388 def flowJobPerfRunList = perfOSList.collect { os ->
389 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, architecture, isPR)}') }"
391 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_${architecture}_flow", isPR, '')) {
394 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
398 // First, build the bits on RHEL7.2
399 b = build(params, '${fullBuildJobName}')
401 // Then, run the perf tests
403 ${flowJobPerfRunList.join(",\n ")}
408 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
409 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
412 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
413 builder.setGithubContext("Linux Perf Test Flow")
414 builder.triggerOnlyOnComment()
415 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
416 builder.triggerForBranch(branch)
417 builder.emitTrigger(newFlowJob)
420 // Set a push trigger
421 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
422 builder.emitTrigger(newFlowJob)
427 def static getDockerImageName(def architecture, def os, def isBuild) {
428 // We must change some docker private images to official later
430 if (architecture == 'arm') {
431 if (os == 'Ubuntu') {
432 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
436 println("Unknown architecture to use docker: ${architecture} ${os}");
440 def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
441 return Utilities.getFullJobName(project, "perf_throughput_${os}_${arch}", isPR)
444 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
445 [true, false].each { isPR ->
446 ['x64','arm'].each { architecture ->
447 def fullBuildJobName = Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_build", isPR)
448 def configuration = 'Release'
451 def crossCompile = ""
452 def python = "python3.5"
454 if (architecture == "arm") {
456 def buildCommands = []
457 def newBuildJob = job(fullBuildJobName) {
458 def additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
459 def dockerImage = getDockerImageName(architecture, 'Ubuntu', true)
460 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
462 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh release ${architecture} cross crosscomponent"
465 buildCommands.each { buildCommand ->
471 azureVMAgentPostBuildAction {
472 agentPostBuildAction('Delete agent after build execution (when idle).')
476 Utilities.setMachineAffinity(newBuildJob, "Ubuntu16.04", 'latest-or-auto')
477 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
478 Utilities.addArchival(newBuildJob, "bin/Product/**")
481 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
482 ['RHEL7.2'].each { os ->
483 def newBuildJob = job(fullBuildJobName) {
485 shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}")
488 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
489 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
490 Utilities.addArchival(newBuildJob, "bin/Product/**")
494 // Actual perf testing on the following OSes
495 def throughputOSList = ['Ubuntu16.04']
496 if (architecture == 'arm') {
497 throughputOSList = ['Ubuntu14.04']
499 def throughputOptLevelList = ['full_opt', 'min_opt']
501 def throughputOSOptLevelList = []
503 throughputOSList.each { os ->
504 throughputOptLevelList.each { opt_level ->
505 throughputOSOptLevelList.add("${os}_${opt_level}")
509 throughputOSList.each { os ->
510 throughputOptLevelList.each { opt_level ->
511 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", architecture, isPR)) {
513 def machineLabel = 'ubuntu_1604_clr_perf'
514 if (architecture == 'arm') {
515 machineLabel = 'ubuntu_1404_clr_perf_arm'
521 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
527 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
532 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
535 def osGroup = getOSGroup(os)
536 def runType = isPR ? 'private' : 'rolling'
537 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
538 def archString = architecture == 'arm' ? ' --arch=arm' : ''
541 shell("bash ./tests/scripts/perf-prep.sh --throughput${archString}")
542 copyArtifacts(fullBuildJobName) {
543 includePatterns("bin/Product/**")
545 buildNumber('\${PRODUCT_BUILD}')
548 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
549 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
550 "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
551 shell("""${python} ./tests/scripts/run-throughput-perf.py \\
552 -arch \"${architecture}\" \\
554 -configuration \"${configuration}\" \\
555 -opt_level \"${opt_level}\" \\
556 -clr_root \"\${WORKSPACE}\" \\
557 -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/lib\" \\
558 -run_type \"${runType}\" \\
559 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
563 // Save machinedata.json to /artifact/bin/ Jenkins dir
564 def archiveSettings = new ArchivalSettings()
565 archiveSettings.addFiles('throughput-*.csv')
566 archiveSettings.addFiles('machinedata.json')
567 archiveSettings.setAlwaysArchive()
568 Utilities.addArchival(newJob, archiveSettings)
570 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
572 // For perf, we need to keep the run results longer
574 // Enable the log rotator
576 artifactDaysToKeep(7)
578 artifactNumToKeep(25)
585 def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
586 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, architecture, isPR)}') }"
588 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_flow", isPR, '')) {
591 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
595 // First, build the bits on RHEL7.2
596 b = build(params, '${fullBuildJobName}')
598 // Then, run the perf tests
600 ${flowJobTPRunList.join(",\n ")}
605 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
606 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
609 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
610 builder.setGithubContext("Linux Throughput Perf Test Flow")
611 builder.triggerOnlyOnComment()
612 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
613 builder.triggerForBranch(branch)
614 builder.emitTrigger(newFlowJob)
617 // Set a push trigger
618 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
619 builder.emitTrigger(newFlowJob)
624 // Setup CoreCLR-Scenarios tests
625 [true, false].each { isPR ->
626 ['Windows_NT'].each { os ->
627 ['x64', 'x86'].each { arch ->
628 ['ryujit'].each { jit ->
629 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
630 def architecture = arch
631 def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
636 label('windows_server_2016_clr_perf')
639 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
645 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
650 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
651 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
654 def configuration = 'Release'
655 def runType = isPR ? 'private' : 'rolling'
656 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
657 def uploadString = '-uploadToBenchview'
661 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
662 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
663 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
665 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
666 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
667 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
668 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
669 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
670 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
671 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
672 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
673 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
675 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
677 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
680 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
683 if (opt_level != 'min_opt') {
684 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
689 def archiveSettings = new ArchivalSettings()
690 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
691 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
692 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
693 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
694 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
695 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
696 archiveSettings.addFiles('machinedata.json')
697 archiveSettings.setAlwaysArchive()
699 Utilities.addArchival(newJob, archiveSettings)
700 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
704 artifactDaysToKeep(14)
706 artifactNumToKeep(100)
718 if (opt_level == 'min_opt') {
719 opts = '\\W+min_opts'
722 if (jit != 'ryujit') {
726 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
727 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
728 builder.triggerOnlyOnComment()
729 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
730 builder.triggerForBranch(branch)
731 builder.emitTrigger(newJob)
733 else if (opt_level == 'full_opt') {
734 // Set a push trigger
735 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
736 builder.emitTrigger(newJob)
739 // Set periodic trigger
740 Utilities.addPeriodicTrigger(newJob, '@daily')
748 // Setup size-on-disk test
749 ['Windows_NT'].each { os ->
750 ['x64', 'x86'].each { arch ->
751 def architecture = arch
752 def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
756 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
760 def channel = 'master'
761 def configuration = 'Release'
762 def runType = 'rolling'
763 def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
764 def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
765 def coreRoot = "${testBin}\\Tests\\Core_Root"
766 def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
769 // Install nuget and get BenchView tools
770 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
771 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
772 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
774 // Generate submission metadata for BenchView
775 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
776 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
777 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
778 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
779 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
780 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
781 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
783 // Generate machine data from BenchView
784 batchFile("py \"${benchViewTools}\\machinedata.py\"")
786 // Build CoreCLR and gnerate test layout
787 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
788 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
790 // Run the size on disk benchmark
791 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
793 // From sodbench.csv, create measurment.json, then submission.json
794 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
795 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
797 // If this is a PR, upload submission.json
798 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
802 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
804 def archiveSettings = new ArchivalSettings()
805 archiveSettings.addFiles('bin/toArchive/**')
806 archiveSettings.addFiles('machinedata.json')
807 archiveSettings.setAlwaysArchive()
809 Utilities.addArchival(newJob, archiveSettings)
810 Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
812 // Set the cron job here. We run nightly on each flavor, regardless of code changes
813 Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
817 artifactDaysToKeep(14)
819 artifactNumToKeep(100)
831 // Setup IlLink tests
832 [true, false].each { isPR ->
833 ['Windows_NT'].each { os ->
834 ['x64'].each { arch ->
835 ['ryujit'].each { jit ->
836 ['full_opt'].each { opt_level ->
837 def architecture = arch
838 def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
843 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
849 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
854 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
855 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
858 def configuration = 'Release'
859 def runType = isPR ? 'private' : 'rolling'
860 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
861 def uploadString = '-uploadToBenchview'
865 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
866 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
867 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
869 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
870 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
871 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
872 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
873 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
874 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
875 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
876 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
877 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
879 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
881 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
884 batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
885 "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
889 def archiveSettings = new ArchivalSettings()
890 archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt')
891 archiveSettings.addFiles('bin/sandbox_logs/**/*.csv')
892 archiveSettings.addFiles('bin/sandbox_logs/**/*.xml')
893 archiveSettings.addFiles('bin/sandbox_logs/**/*.log')
894 archiveSettings.addFiles('bin/sandbox_logs/**/*.md')
895 archiveSettings.addFiles('bin/sandbox_logs/**/*.etl')
896 archiveSettings.addFiles('machinedata.json')
897 archiveSettings.setAlwaysArchive()
899 // Set the label (currently we are only measuring size, therefore we are running on VM).
900 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
901 Utilities.addArchival(newJob, archiveSettings)
902 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
906 artifactDaysToKeep(14)
908 artifactNumToKeep(100)
919 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
920 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
921 builder.triggerOnlyOnComment()
922 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
923 builder.triggerForBranch(branch)
924 builder.emitTrigger(newJob)
927 // Set a push trigger
928 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
929 builder.emitTrigger(newJob)
937 Utilities.createHelperJob(this, project, branch,
938 "Welcome to the ${project} Perf help",