1 // Import the utility functionality.
3 import jobs.generation.*;
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
10 def static getOSGroup(def os) {
11 def osGroupMap = ['Ubuntu14.04':'Linux',
13 'Ubuntu16.04': 'Linux',
17 'Windows_NT':'Windows_NT',
20 'OpenSUSE13.2': 'Linux',
21 'OpenSUSE42.1': 'Linux',
22 'LinuxARMEmulator': 'Linux']
23 def osGroup = osGroupMap.get(os, null)
24 assert osGroup != null : "Could not find os group for ${os}"
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30 ['Windows_NT'].each { os ->
31 ['x64', 'x86'].each { arch ->
32 [true, false].each { isSmoketest ->
33 def architecture = arch
34 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_smoketest" : "perf_perflab_${os}_${arch}"
38 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
41 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
43 label('windows_clr_perf')
46 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
54 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
61 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
62 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
69 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
70 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
73 def configuration = 'Release'
74 def runType = isPR ? 'private' : 'rolling'
75 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
76 def uploadString = isSmoketest ? '' : '-uploadToBenchview'
81 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
82 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
83 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
84 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
85 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
86 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
87 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
88 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
89 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
90 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
91 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
92 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
94 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
96 // Run with just stopwatch: Profile=Off
97 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
98 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
100 // Run with the full set of counters enabled: Profile=On
101 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
102 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
108 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
110 // Save machinedata.json to /artifact/bin/ Jenkins dir
111 def archiveSettings = new ArchivalSettings()
112 archiveSettings.addFiles('Perf-*.xml')
113 archiveSettings.addFiles('Perf-*.etl')
114 archiveSettings.addFiles('Perf-*.log')
115 archiveSettings.addFiles('machinedata.json')
116 Utilities.addArchival(newJob, archiveSettings)
118 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
129 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
132 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests Correctness")
136 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
137 builder.triggerOnlyOnComment()
138 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
140 builder.triggerForBranch(branch)
141 builder.emitTrigger(newJob)
144 // Set a push trigger
145 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
146 builder.emitTrigger(newJob)
153 // Setup throughput perflab tests runs
154 [true, false].each { isPR ->
155 ['Windows_NT'].each { os ->
156 ['x64', 'x86'].each { arch ->
157 ['full_opt', 'min_opt'].each { opt_level ->
158 def architecture = arch
160 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}", isPR)) {
162 label('windows_clr_perf')
165 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
173 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
176 def configuration = 'Release'
177 def runType = isPR ? 'private' : 'rolling'
178 def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
183 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
184 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
185 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
186 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
187 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
188 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
189 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
190 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
191 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
192 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
193 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
194 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
195 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
196 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
197 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
201 // Save machinedata.json to /artifact/bin/ Jenkins dir
202 def archiveSettings = new ArchivalSettings()
203 archiveSettings.addFiles('throughput-*.csv')
204 Utilities.addArchival(newJob, archiveSettings)
206 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
210 if (opt_level == 'min_opts')
212 opts = '\\W+min_opts'
214 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
215 builder.setGithubContext("${os} ${arch} ${opt_level} CoreCLR Throughput Perf Tests")
216 builder.triggerOnlyOnComment()
217 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}\\W+throughput.*")
218 builder.triggerForBranch(branch)
219 builder.emitTrigger(newJob)
222 // Set a push trigger
223 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
224 builder.emitTrigger(newJob)
231 def static getFullPerfJobName(def project, def os, def isPR) {
232 return Utilities.getFullJobName(project, "perf_${os}", isPR)
235 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
236 [true, false].each { isPR ->
237 def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
238 def architecture = 'x64'
239 def configuration = 'Release'
241 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
242 ['RHEL7.2'].each { os ->
243 def newBuildJob = job(fullBuildJobName) {
245 shell("./build.sh verbose ${architecture} ${configuration}")
248 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
249 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
250 Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
253 // Actual perf testing on the following OSes
254 def perfOSList = ['Ubuntu14.04']
255 perfOSList.each { os ->
256 def newJob = job(getFullPerfJobName(project, os, isPR)) {
258 label('linux_clr_perf')
261 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
269 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
274 // Cap the maximum number of iterations to 21.
275 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
276 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
277 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
280 def osGroup = getOSGroup(os)
281 def runType = isPR ? 'private' : 'rolling'
282 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
285 shell("./tests/scripts/perf-prep.sh")
286 shell("./init-tools.sh")
287 copyArtifacts(fullBuildJobName) {
288 includePatterns("bin/**")
290 buildNumber('\${PRODUCT_BUILD}')
293 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
294 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
295 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
296 shell("""./tests/scripts/run-xunit-perf.sh \\
297 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
298 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
299 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
300 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
301 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
302 --runType=\"${runType}\" \\
303 --benchViewOS=\"${os}\" \\
304 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
305 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
306 --uploadToBenchview""")
310 // Save machinedata.json to /artifact/bin/ Jenkins dir
311 def archiveSettings = new ArchivalSettings()
312 archiveSettings.addFiles('Perf-*.log')
313 archiveSettings.addFiles('Perf-*.xml')
314 archiveSettings.addFiles('machinedata.json')
315 Utilities.addArchival(newJob, archiveSettings)
317 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
319 // For perf, we need to keep the run results longer
321 // Enable the log rotator
323 artifactDaysToKeep(7)
325 artifactNumToKeep(25)
331 def flowJobPerfRunList = perfOSList.collect { os ->
332 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
334 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
337 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
341 // First, build the bits on RHEL7.2
342 b = build(params, '${fullBuildJobName}')
344 // Then, run the perf tests
346 ${flowJobPerfRunList.join(",\n ")}
351 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
352 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
355 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
356 builder.setGithubContext("Linux Perf Test Flow")
357 builder.triggerOnlyOnComment()
358 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
359 builder.triggerForBranch(branch)
360 builder.emitTrigger(newFlowJob)
363 // Set a push trigger
364 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
365 builder.emitTrigger(newFlowJob)
370 def static getFullThroughputJobName(def project, def os, def isPR) {
371 return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
374 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
375 [true, false].each { isPR ->
376 def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
377 def architecture = 'x64'
378 def configuration = 'Release'
380 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
381 ['RHEL7.2'].each { os ->
382 def newBuildJob = job(fullBuildJobName) {
384 shell("./build.sh verbose ${architecture} ${configuration}")
387 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
388 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
389 Utilities.addArchival(newBuildJob, "bin/Product/**")
392 // Actual perf testing on the following OSes
393 def throughputOSList = ['Ubuntu14.04']
394 def throughputOptLevelList = ['full_opt', 'min_opt']
396 def throughputOSOptLevelList = []
398 throughputOSList.each { os ->
399 throughputOptLevelList.each { opt_level ->
400 throughputOSOptLevelList.add("${os}_${opt_level}")
404 throughputOSList.each { os ->
405 throughputOptLevelList.each { opt_level ->
406 def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
408 label('linux_clr_perf')
411 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
419 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
424 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
427 def osGroup = getOSGroup(os)
428 def runType = isPR ? 'private' : 'rolling'
429 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
432 shell("bash ./tests/scripts/perf-prep.sh --throughput")
433 shell("./init-tools.sh")
434 copyArtifacts(fullBuildJobName) {
435 includePatterns("bin/Product/**")
437 buildNumber('\${PRODUCT_BUILD}')
440 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
441 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
442 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
443 shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
444 -arch \"${architecture}\" \\
446 -configuration \"${configuration}\" \\
447 -opt_level \"${opt_level}\" \\
448 -clr_root \"\${WORKSPACE}\" \\
449 -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
450 -run_type \"${runType}\" \\
451 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
455 // Save machinedata.json to /artifact/bin/ Jenkins dir
456 def archiveSettings = new ArchivalSettings()
457 archiveSettings.addFiles('throughput-*.csv')
458 archiveSettings.addFiles('machinedata.json')
459 Utilities.addArchival(newJob, archiveSettings)
461 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
463 // For perf, we need to keep the run results longer
465 // Enable the log rotator
467 artifactDaysToKeep(7)
469 artifactNumToKeep(25)
476 def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
477 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
479 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
482 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
486 // First, build the bits on RHEL7.2
487 b = build(params, '${fullBuildJobName}')
489 // Then, run the perf tests
491 ${flowJobTPRunList.join(",\n ")}
496 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
497 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
500 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
501 builder.setGithubContext("Linux Throughput Perf Test Flow")
502 builder.triggerOnlyOnComment()
503 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
504 builder.triggerForBranch(branch)
505 builder.emitTrigger(newFlowJob)
508 // Set a push trigger
509 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
510 builder.emitTrigger(newFlowJob)
515 // Setup CoreCLR-Scenarios tests
516 [true, false].each { isPR ->
517 ['Windows_NT'].each { os ->
518 ['x64', 'x86'].each { arch ->
519 def architecture = arch
520 def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}", isPR)) {
522 label('windows_clr_perf')
525 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
533 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
539 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
540 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
542 def configuration = 'Release'
543 def runType = isPR ? 'private' : 'rolling'
544 def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
545 def uploadString = '-uploadToBenchview'
549 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
550 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
551 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
553 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
554 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
555 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
556 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
557 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
558 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
559 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
560 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
561 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
563 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
565 // Scenario: JitBench
566 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -scenarioTest -group CoreCLR-Scenarios")
570 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -nowarmup -runtype ${runType} -scenarioTest -group ILLink")
575 // Save machinedata.json to /artifact/bin/ Jenkins dir
576 def archiveSettings = new ArchivalSettings()
577 archiveSettings.addFiles('Perf-*.xml')
578 archiveSettings.addFiles('Perf-*.log')
579 archiveSettings.addFiles('machinedata.json')
580 Utilities.addArchival(newJob, archiveSettings)
582 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
593 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
594 builder.setGithubContext("${os} ${arch} Performance Scenarios Tests")
595 builder.triggerOnlyOnComment()
596 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf\\W+scenarios.*")
597 builder.triggerForBranch(branch)
598 builder.emitTrigger(newJob)
601 // Set a push trigger
602 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
603 builder.emitTrigger(newJob)
609 Utilities.createHelperJob(this, project, branch,
610 "Welcome to the ${project} Perf help",