1 // Import the utility functionality.
3 import jobs.generation.*;
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
10 def static getOSGroup(def os) {
11 def osGroupMap = ['Ubuntu14.04':'Linux',
13 'Ubuntu16.04': 'Linux',
17 'Windows_NT':'Windows_NT',
20 'OpenSUSE13.2': 'Linux',
21 'OpenSUSE42.1': 'Linux',
22 'LinuxARMEmulator': 'Linux']
23 def osGroup = osGroupMap.get(os, null)
24 assert osGroup != null : "Could not find os group for ${os}"
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30 ['Windows_NT'].each { os ->
31 ['x64', 'x86'].each { arch ->
32 ['ryujit'].each { jit ->
33 ['full_opt', 'min_opt'].each { opt_level ->
35 def architecture = arch
36 def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
39 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
41 label('windows_server_2016_clr_perf')
44 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
50 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
55 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
56 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
59 def configuration = 'Release'
60 def runType = isPR ? 'private' : 'rolling'
61 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
62 def uploadString = '-uploadToBenchview'
67 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
68 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
69 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
70 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
71 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
72 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
73 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
74 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
75 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
76 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
77 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
78 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
80 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
82 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
84 // Run with just stopwatch: Profile=Off
85 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
86 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
88 // Run with the full set of counters enabled: Profile=On
89 if (opt_level != 'min_opt') {
90 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
91 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
96 def archiveSettings = new ArchivalSettings()
97 archiveSettings.addFiles('bin/sandbox_logs/**')
98 archiveSettings.addFiles('machinedata.json')
100 Utilities.addArchival(newJob, archiveSettings)
101 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
105 artifactDaysToKeep(30)
107 artifactNumToKeep(200)
118 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
119 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
122 if (opt_level == 'min_opt') {
123 opts = '\\W+min_opts'
126 if (jit != 'ryujit') {
130 builder.triggerOnlyOnComment()
131 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
132 builder.triggerForBranch(branch)
133 builder.emitTrigger(newJob)
135 else if (opt_level == 'full_opt') {
136 // Set a push trigger
137 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
138 builder.emitTrigger(newJob)
141 // Set periodic trigger
142 Utilities.addPeriodicTrigger(newJob, '@daily')
150 // Setup throughput perflab tests runs
151 [true, false].each { isPR ->
152 ['Windows_NT'].each { os ->
153 ['x64', 'x86'].each { arch ->
154 ['ryujit'].each { jit ->
155 [true, false].each { pgo_optimized ->
156 ['full_opt', 'min_opt'].each { opt_level ->
157 def architecture = arch
162 if (!pgo_optimized) {
163 pgo_build = " -nopgooptimize"
168 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
170 label('windows_server_2016_clr_perf')
173 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
179 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
183 def configuration = 'Release'
184 def runType = isPR ? 'private' : 'rolling'
185 def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
189 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
190 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
191 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
192 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
193 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
194 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
195 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
196 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
197 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
198 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
199 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
200 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
201 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
202 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
203 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
207 // Save machinedata.json to /artifact/bin/ Jenkins dir
208 def archiveSettings = new ArchivalSettings()
209 archiveSettings.addFiles('throughput-*.csv')
210 Utilities.addArchival(newJob, archiveSettings)
212 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
216 if (opt_level == 'min_opt') {
217 opts = '\\W+min_opts'
221 if (jit != 'ryujit') {
227 pgo_trigger = "\\W+nopgo"
231 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
232 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
233 builder.triggerOnlyOnComment()
234 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
235 builder.triggerForBranch(branch)
236 builder.emitTrigger(newJob)
238 else if (opt_level == 'full_opt' && pgo_optimized) {
239 // Set a push trigger
240 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
241 builder.emitTrigger(newJob)
244 // Set periodic trigger
245 Utilities.addPeriodicTrigger(newJob, '@daily')
254 def static getFullPerfJobName(def project, def os, def isPR) {
255 return Utilities.getFullJobName(project, "perf_${os}", isPR)
258 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
259 [true, false].each { isPR ->
260 def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
261 def architecture = 'x64'
262 def configuration = 'Release'
264 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
265 ['RHEL7.2'].each { os ->
266 def newBuildJob = job(fullBuildJobName) {
268 shell("./build.sh verbose ${architecture} ${configuration}")
269 shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}")
272 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
273 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
274 Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
278 // Actual perf testing on the following OSes
279 def perfOSList = ['Ubuntu16.04']
280 perfOSList.each { os ->
281 def newJob = job(getFullPerfJobName(project, os, isPR)) {
283 label('ubuntu_1604_clr_perf')
286 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
292 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
297 // Cap the maximum number of iterations to 21.
298 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
299 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
300 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
303 def osGroup = getOSGroup(os)
304 def runType = isPR ? 'private' : 'rolling'
305 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
306 def uploadString = '-uploadToBenchview'
308 def runXUnitCommonArgs = "-arch ${architecture} -os Ubuntu16.04 -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
311 shell("./tests/scripts/perf-prep.sh --nocorefx")
312 shell("./init-tools.sh")
313 copyArtifacts(fullBuildJobName) {
314 includePatterns("bin/**")
316 buildNumber('\${PRODUCT_BUILD}')
319 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
320 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
321 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
322 shell("""python3 ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
326 def archiveSettings = new ArchivalSettings()
327 archiveSettings.addFiles('bin/sandbox_logs/**')
328 archiveSettings.addFiles('machinedata.json')
330 Utilities.addArchival(newJob, archiveSettings)
331 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
333 // For perf, we need to keep the run results longer
335 // Enable the log rotator
337 artifactDaysToKeep(30)
339 artifactNumToKeep(200)
350 def flowJobPerfRunList = perfOSList.collect { os ->
351 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
353 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
356 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
360 // First, build the bits on RHEL7.2
361 b = build(params, '${fullBuildJobName}')
363 // Then, run the perf tests
365 ${flowJobPerfRunList.join(",\n ")}
370 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
371 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
374 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
375 builder.setGithubContext("Linux Perf Test Flow")
376 builder.triggerOnlyOnComment()
377 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
378 builder.triggerForBranch(branch)
379 builder.emitTrigger(newFlowJob)
382 // Set a push trigger
383 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
384 builder.emitTrigger(newFlowJob)
389 def static getFullThroughputJobName(def project, def os, def isPR) {
390 return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
393 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
394 [true, false].each { isPR ->
395 def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
396 def architecture = 'x64'
397 def configuration = 'Release'
399 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
400 ['RHEL7.2'].each { os ->
401 def newBuildJob = job(fullBuildJobName) {
403 shell("./build.sh verbose ${architecture} ${configuration}")
406 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
407 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
408 Utilities.addArchival(newBuildJob, "bin/Product/**")
411 // Actual perf testing on the following OSes
412 def throughputOSList = ['Ubuntu14.04']
413 def throughputOptLevelList = ['full_opt', 'min_opt']
415 def throughputOSOptLevelList = []
417 throughputOSList.each { os ->
418 throughputOptLevelList.each { opt_level ->
419 throughputOSOptLevelList.add("${os}_${opt_level}")
423 throughputOSList.each { os ->
424 throughputOptLevelList.each { opt_level ->
425 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
427 label('ubuntu_1604_clr_perf')
430 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
436 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
441 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
444 def osGroup = getOSGroup(os)
445 def runType = isPR ? 'private' : 'rolling'
446 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
449 shell("bash ./tests/scripts/perf-prep.sh --throughput")
450 shell("./init-tools.sh")
451 copyArtifacts(fullBuildJobName) {
452 includePatterns("bin/Product/**")
454 buildNumber('\${PRODUCT_BUILD}')
457 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
458 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
459 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
460 shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
461 -arch \"${architecture}\" \\
463 -configuration \"${configuration}\" \\
464 -opt_level \"${opt_level}\" \\
465 -clr_root \"\${WORKSPACE}\" \\
466 -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
467 -run_type \"${runType}\" \\
468 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
472 // Save machinedata.json to /artifact/bin/ Jenkins dir
473 def archiveSettings = new ArchivalSettings()
474 archiveSettings.addFiles('throughput-*.csv')
475 archiveSettings.addFiles('machinedata.json')
476 Utilities.addArchival(newJob, archiveSettings)
478 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
480 // For perf, we need to keep the run results longer
482 // Enable the log rotator
484 artifactDaysToKeep(7)
486 artifactNumToKeep(25)
493 def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
494 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
496 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
499 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
503 // First, build the bits on RHEL7.2
504 b = build(params, '${fullBuildJobName}')
506 // Then, run the perf tests
508 ${flowJobTPRunList.join(",\n ")}
513 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
514 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
517 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
518 builder.setGithubContext("Linux Throughput Perf Test Flow")
519 builder.triggerOnlyOnComment()
520 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
521 builder.triggerForBranch(branch)
522 builder.emitTrigger(newFlowJob)
525 // Set a push trigger
526 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
527 builder.emitTrigger(newFlowJob)
532 // Setup CoreCLR-Scenarios tests
533 [true, false].each { isPR ->
534 ['Windows_NT'].each { os ->
535 ['x64', 'x86'].each { arch ->
536 ['ryujit'].each { jit ->
537 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
538 def architecture = arch
539 def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
544 label('windows_server_2016_clr_perf')
547 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
553 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
558 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
559 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
562 def configuration = 'Release'
563 def runType = isPR ? 'private' : 'rolling'
564 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
565 def uploadString = '-uploadToBenchview'
569 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
570 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
571 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
573 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
574 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
575 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
576 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
577 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
578 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
579 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
580 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
581 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
583 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
585 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
588 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
591 if (opt_level != 'min_opt') {
592 batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
597 def archiveSettings = new ArchivalSettings()
598 archiveSettings.addFiles('bin/sandbox_logs/**')
599 archiveSettings.addFiles('machinedata.json')
601 Utilities.addArchival(newJob, archiveSettings)
602 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
606 artifactDaysToKeep(30)
608 artifactNumToKeep(200)
620 if (opt_level == 'min_opt') {
621 opts = '\\W+min_opts'
624 if (jit != 'ryujit') {
628 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
629 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
630 builder.triggerOnlyOnComment()
631 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
632 builder.triggerForBranch(branch)
633 builder.emitTrigger(newJob)
635 else if (opt_level == 'full_opt') {
636 // Set a push trigger
637 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
638 builder.emitTrigger(newJob)
641 // Set periodic trigger
642 Utilities.addPeriodicTrigger(newJob, '@daily')
650 // Setup size-on-disk test
651 ['Windows_NT'].each { os ->
652 ['x64', 'x86'].each { arch ->
653 def architecture = arch
654 def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
658 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
662 def channel = 'master'
663 def configuration = 'Release'
664 def runType = 'rolling'
665 def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
666 def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
667 def coreRoot = "${testBin}\\Tests\\Core_Root"
668 def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
671 // Install nuget and get BenchView tools
672 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
673 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
674 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
676 // Generate submission metadata for BenchView
677 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
678 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
679 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
680 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
681 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
682 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
683 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
685 // Generate machine data from BenchView
686 batchFile("py \"${benchViewTools}\\machinedata.py\"")
688 // Build CoreCLR and gnerate test layout
689 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
690 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
692 // Run the size on disk benchmark
693 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
695 // From sodbench.csv, create measurment.json, then submission.json
696 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
697 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
699 // If this is a PR, upload submission.json
700 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
704 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
706 def archiveSettings = new ArchivalSettings()
707 archiveSettings.addFiles('bin/toArchive/**')
708 archiveSettings.addFiles('machinedata.json')
710 Utilities.addArchival(newJob, archiveSettings)
711 Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
713 // Set the cron job here. We run nightly on each flavor, regardless of code changes
714 Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
718 artifactDaysToKeep(30)
720 artifactNumToKeep(200)
732 // Setup IlLink tests
733 [true, false].each { isPR ->
734 ['Windows_NT'].each { os ->
735 ['x64'].each { arch ->
736 ['ryujit'].each { jit ->
737 ['full_opt'].each { opt_level ->
738 def architecture = arch
739 def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
744 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
750 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
755 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
756 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
759 def configuration = 'Release'
760 def runType = isPR ? 'private' : 'rolling'
761 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
762 def uploadString = '-uploadToBenchview'
766 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
767 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
768 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
770 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
771 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
772 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
773 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
774 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
775 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
776 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
777 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
778 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
780 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
782 def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
785 batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
786 "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
790 def archiveSettings = new ArchivalSettings()
791 archiveSettings.addFiles('bin/sandbox_logs/**')
792 archiveSettings.addFiles('machinedata.json')
794 // Set the label (currently we are only measuring size, therefore we are running on VM).
795 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
796 Utilities.addArchival(newJob, archiveSettings)
797 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
801 artifactDaysToKeep(30)
803 artifactNumToKeep(200)
814 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
815 builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
816 builder.triggerOnlyOnComment()
817 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
818 builder.triggerForBranch(branch)
819 builder.emitTrigger(newJob)
822 // Set a push trigger
823 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
824 builder.emitTrigger(newJob)
832 Utilities.createHelperJob(this, project, branch,
833 "Welcome to the ${project} Perf help",