1 // Import the utility functionality.
3 import jobs.generation.*;
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
10 def static getOSGroup(def os) {
11 def osGroupMap = ['Ubuntu14.04':'Linux',
13 'Ubuntu16.04': 'Linux',
17 'Windows_NT':'Windows_NT',
20 'OpenSUSE13.2': 'Linux',
21 'OpenSUSE42.1': 'Linux',
22 'LinuxARMEmulator': 'Linux']
23 def osGroup = osGroupMap.get(os, null)
24 assert osGroup != null : "Could not find os group for ${os}"
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30 ['Windows_NT'].each { os ->
31 ['x64', 'x86'].each { arch ->
32 [true, false].each { isSmoketest ->
33 ['ryujit'].each { jit ->
35 if (arch == 'x64' && jit == 'legacy_backend') {
39 ['full_opt', 'min_opt'].each { opt_level ->
40 if (isSmoketest && opt_level == 'min_opt') {
44 def architecture = arch
45 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
48 if (jit == 'legacy_backend') {
49 testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
52 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
54 label('windows_server_2016_clr_perf')
57 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
63 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
68 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
69 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
74 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
75 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
79 def configuration = 'Release'
80 def runType = isPR ? 'private' : 'rolling'
81 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
82 def uploadString = isSmoketest ? '' : '-uploadToBenchview'
87 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
88 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
89 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
90 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
91 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
92 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
93 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
94 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
95 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
96 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
97 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
98 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
100 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
102 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
104 // Run with just stopwatch: Profile=Off
105 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
106 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\Off\\")
108 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
109 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\")
111 // Run with the full set of counters enabled: Profile=On
112 if (opt_level != 'min_opt') {
113 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
114 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
116 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
117 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
123 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
126 def archiveSettings = new ArchivalSettings()
127 archiveSettings.addFiles('bin/toArchive/**')
128 archiveSettings.addFiles('machinedata.json')
130 Utilities.addArchival(newJob, archiveSettings)
131 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
135 artifactDaysToKeep(30)
137 artifactNumToKeep(200)
148 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
150 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
153 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
156 if (opt_level == 'min_opt') {
157 opts = '\\W+min_opts'
160 if (jit != 'ryujit') {
164 builder.triggerOnlyOnComment()
165 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
167 builder.triggerForBranch(branch)
168 builder.emitTrigger(newJob)
171 // Set a push trigger
172 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
173 builder.emitTrigger(newJob)
182 // Setup throughput perflab tests runs
183 [true, false].each { isPR ->
184 ['Windows_NT'].each { os ->
185 ['x64', 'x86'].each { arch ->
186 ['ryujit', 'legacy_backend'].each { jit ->
187 [true, false].each { pgo_optimized ->
188 if (arch == 'x64' && jit == 'legacy_backend') {
192 // pgo not supported for legacy_backend
193 if (pgo_optimized && jit == 'legacy_backend') {
197 ['full_opt', 'min_opt'].each { opt_level ->
198 def architecture = arch
203 if (!pgo_optimized) {
204 pgo_build = " -nopgooptimize"
209 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
211 label('windows_server_2016_clr_perf')
214 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
220 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
224 def configuration = 'Release'
225 def runType = isPR ? 'private' : 'rolling'
226 def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
230 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
231 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
232 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
233 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
234 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
235 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
236 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
237 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
238 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
239 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
240 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
241 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
242 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
243 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
244 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
248 // Save machinedata.json to /artifact/bin/ Jenkins dir
249 def archiveSettings = new ArchivalSettings()
250 archiveSettings.addFiles('throughput-*.csv')
251 Utilities.addArchival(newJob, archiveSettings)
253 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
257 if (opt_level == 'min_opt') {
258 opts = '\\W+min_opts'
262 if (jit != 'ryujit') {
268 pgo_trigger = "\\W+nopgo"
272 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
273 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
274 builder.triggerOnlyOnComment()
275 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
276 builder.triggerForBranch(branch)
277 builder.emitTrigger(newJob)
280 // Set a push trigger
281 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
282 builder.emitTrigger(newJob)
291 def static getFullPerfJobName(def project, def os, def isPR) {
292 return Utilities.getFullJobName(project, "perf_${os}", isPR)
295 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
296 [true, false].each { isPR ->
297 def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
298 def architecture = 'x64'
299 def configuration = 'Release'
301 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
302 ['RHEL7.2'].each { os ->
303 def newBuildJob = job(fullBuildJobName) {
305 shell("./build.sh verbose ${architecture} ${configuration}")
308 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
309 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
310 Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
314 // Actual perf testing on the following OSes
315 def perfOSList = ['Ubuntu14.04']
316 perfOSList.each { os ->
317 def newJob = job(getFullPerfJobName(project, os, isPR)) {
319 label('linux_clr_perf')
322 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
328 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
333 // Cap the maximum number of iterations to 21.
334 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
335 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
336 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
339 def osGroup = getOSGroup(os)
340 def runType = isPR ? 'private' : 'rolling'
341 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
344 shell("./tests/scripts/perf-prep.sh")
345 shell("./init-tools.sh")
346 copyArtifacts(fullBuildJobName) {
347 includePatterns("bin/**")
349 buildNumber('\${PRODUCT_BUILD}')
352 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
353 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
354 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
355 shell("""./tests/scripts/run-xunit-perf.sh \\
356 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
357 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
358 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
359 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
360 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
361 --runType=\"${runType}\" \\
362 --benchViewOS=\"${os}\" \\
363 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
364 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
365 --uploadToBenchview""")
366 shell("mkdir -p bin/toArchive/sandbox/Logs/")
367 shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
371 def archiveSettings = new ArchivalSettings()
372 archiveSettings.addFiles('bin/toArchive/**')
373 archiveSettings.addFiles('machinedata.json')
375 Utilities.addArchival(newJob, archiveSettings)
376 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
378 // For perf, we need to keep the run results longer
380 // Enable the log rotator
382 artifactDaysToKeep(30)
384 artifactNumToKeep(200)
395 def flowJobPerfRunList = perfOSList.collect { os ->
396 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
398 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
401 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
405 // First, build the bits on RHEL7.2
406 b = build(params, '${fullBuildJobName}')
408 // Then, run the perf tests
410 ${flowJobPerfRunList.join(",\n ")}
415 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
416 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
419 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
420 builder.setGithubContext("Linux Perf Test Flow")
421 builder.triggerOnlyOnComment()
422 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
423 builder.triggerForBranch(branch)
424 builder.emitTrigger(newFlowJob)
427 // Set a push trigger
428 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
429 builder.emitTrigger(newFlowJob)
434 def static getFullThroughputJobName(def project, def os, def isPR) {
435 return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
438 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
439 [true, false].each { isPR ->
440 def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
441 def architecture = 'x64'
442 def configuration = 'Release'
444 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
445 ['RHEL7.2'].each { os ->
446 def newBuildJob = job(fullBuildJobName) {
448 shell("./build.sh verbose ${architecture} ${configuration}")
451 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
452 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
453 Utilities.addArchival(newBuildJob, "bin/Product/**")
456 // Actual perf testing on the following OSes
457 def throughputOSList = ['Ubuntu14.04']
458 def throughputOptLevelList = ['full_opt', 'min_opt']
460 def throughputOSOptLevelList = []
462 throughputOSList.each { os ->
463 throughputOptLevelList.each { opt_level ->
464 throughputOSOptLevelList.add("${os}_${opt_level}")
468 throughputOSList.each { os ->
469 throughputOptLevelList.each { opt_level ->
470 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
472 label('linux_clr_perf')
475 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
481 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
486 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
489 def osGroup = getOSGroup(os)
490 def runType = isPR ? 'private' : 'rolling'
491 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
494 shell("bash ./tests/scripts/perf-prep.sh --throughput")
495 shell("./init-tools.sh")
496 copyArtifacts(fullBuildJobName) {
497 includePatterns("bin/Product/**")
499 buildNumber('\${PRODUCT_BUILD}')
502 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
503 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
504 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
505 shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
506 -arch \"${architecture}\" \\
508 -configuration \"${configuration}\" \\
509 -opt_level \"${opt_level}\" \\
510 -clr_root \"\${WORKSPACE}\" \\
511 -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
512 -run_type \"${runType}\" \\
513 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
517 // Save machinedata.json to /artifact/bin/ Jenkins dir
518 def archiveSettings = new ArchivalSettings()
519 archiveSettings.addFiles('throughput-*.csv')
520 archiveSettings.addFiles('machinedata.json')
521 Utilities.addArchival(newJob, archiveSettings)
523 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
525 // For perf, we need to keep the run results longer
527 // Enable the log rotator
529 artifactDaysToKeep(7)
531 artifactNumToKeep(25)
538 def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
539 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
541 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
544 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
548 // First, build the bits on RHEL7.2
549 b = build(params, '${fullBuildJobName}')
551 // Then, run the perf tests
553 ${flowJobTPRunList.join(",\n ")}
558 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
559 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
562 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
563 builder.setGithubContext("Linux Throughput Perf Test Flow")
564 builder.triggerOnlyOnComment()
565 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
566 builder.triggerForBranch(branch)
567 builder.emitTrigger(newFlowJob)
570 // Set a push trigger
571 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
572 builder.emitTrigger(newFlowJob)
577 // Setup CoreCLR-Scenarios tests
578 [true, false].each { isPR ->
579 ['Windows_NT'].each { os ->
580 ['x64', 'x86'].each { arch ->
581 ['ryujit'].each { jit ->
582 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
583 def architecture = arch
584 def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
589 label('windows_server_2016_clr_perf')
592 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
598 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
603 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
604 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
607 def configuration = 'Release'
608 def runType = isPR ? 'private' : 'rolling'
609 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
610 def uploadString = '-uploadToBenchview'
614 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
615 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
616 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
618 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
619 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
620 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
621 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
622 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
623 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
624 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
625 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
626 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
628 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
630 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
633 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
634 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\Off\\")
637 if (opt_level != 'min_opt') {
638 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
639 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\On\\")
644 def archiveSettings = new ArchivalSettings()
645 archiveSettings.addFiles('bin/toArchive/**')
646 archiveSettings.addFiles('machinedata.json')
648 Utilities.addArchival(newJob, archiveSettings)
649 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
653 artifactDaysToKeep(30)
655 artifactNumToKeep(200)
667 if (opt_level == 'min_opt') {
668 opts = '\\W+min_opts'
671 if (jit != 'ryujit') {
675 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
676 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
677 builder.triggerOnlyOnComment()
678 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
679 builder.triggerForBranch(branch)
680 builder.emitTrigger(newJob)
683 // Set a push trigger
684 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
685 builder.emitTrigger(newJob)
693 // Setup size-on-disk test
694 ['Windows_NT'].each { os ->
695 ['x64', 'x86'].each { arch ->
696 def architecture = arch
697 def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
701 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
705 def channel = 'master'
706 def configuration = 'Release'
707 def runType = 'rolling'
708 def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
709 def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
710 def coreRoot = "${testBin}\\Tests\\Core_Root"
711 def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
714 // Install nuget and get BenchView tools
715 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
716 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
717 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
719 // Generate submission metadata for BenchView
720 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
721 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
722 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
723 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
724 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
725 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
726 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
728 // Generate machine data from BenchView
729 batchFile("py \"${benchViewTools}\\machinedata.py\"")
731 // Build CoreCLR and gnerate test layout
732 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
733 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
735 // Run the size on disk benchmark
736 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
738 // From sodbench.csv, create measurment.json, then submission.json
739 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
740 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
742 // If this is a PR, upload submission.json
743 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
747 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
749 def archiveSettings = new ArchivalSettings()
750 archiveSettings.addFiles('bin/toArchive/**')
751 archiveSettings.addFiles('machinedata.json')
753 Utilities.addArchival(newJob, archiveSettings)
754 Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
756 // Set the cron job here. We run nightly on each flavor, regardless of code changes
757 Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
761 artifactDaysToKeep(30)
763 artifactNumToKeep(200)
775 // Setup IlLink tests
776 [true, false].each { isPR ->
777 ['Windows_NT'].each { os ->
778 ['x64'].each { arch ->
779 ['ryujit'].each { jit ->
780 ['full_opt'].each { opt_level ->
781 def architecture = arch
782 def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
787 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
793 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
798 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
799 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
802 def configuration = 'Release'
803 def runType = isPR ? 'private' : 'rolling'
804 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
805 def uploadString = '-uploadToBenchview'
809 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
810 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
811 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
813 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
814 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
815 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
816 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
817 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
818 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
819 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
820 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
821 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
823 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
825 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
828 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
829 batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
833 def archiveSettings = new ArchivalSettings()
834 archiveSettings.addFiles('bin/toArchive/**')
835 archiveSettings.addFiles('machinedata.json')
837 // Set the label (currently we are only measuring size, therefore we are running on VM).
838 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
839 Utilities.addArchival(newJob, archiveSettings)
840 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
844 artifactDaysToKeep(30)
846 artifactNumToKeep(200)
857 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
858 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
859 builder.triggerOnlyOnComment()
860 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
861 builder.triggerForBranch(branch)
862 builder.emitTrigger(newJob)
865 // Set a push trigger
866 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
867 builder.emitTrigger(newJob)
875 Utilities.createHelperJob(this, project, branch,
876 "Welcome to the ${project} Perf help",