1 // Import the utility functionality.
3 import jobs.generation.*;
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
10 def static getOSGroup(def os) {
11 def osGroupMap = ['Ubuntu14.04':'Linux',
13 'Ubuntu16.04': 'Linux',
17 'Windows_NT':'Windows_NT',
20 'OpenSUSE13.2': 'Linux',
21 'OpenSUSE42.1': 'Linux',
22 'LinuxARMEmulator': 'Linux']
23 def osGroup = osGroupMap.get(os, null)
24 assert osGroup != null : "Could not find os group for ${os}"
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30 ['Windows_NT'].each { os ->
31 ['x64', 'x86'].each { arch ->
32 [true, false].each { isSmoketest ->
33 def architecture = arch
34 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_smoketest" : "perf_perflab_${os}_${arch}"
36 if (arch == 'x86jit32')
39 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
41 else if (arch == 'x86')
43 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
46 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
48 label('windows_clr_perf')
51 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
59 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
66 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
67 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
74 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
75 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
78 def configuration = 'Release'
79 def runType = isPR ? 'private' : 'rolling'
80 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
81 def uploadString = isSmoketest ? '' : '-uploadToBenchview'
86 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
87 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
88 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
89 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
90 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
91 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
92 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
93 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
94 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
95 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
96 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
97 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
99 if (arch == 'x86jit32')
101 // Download package and copy compatjit into Core_Root
102 batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
103 "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
106 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
108 // Run with just stopwatch: Profile=Off
109 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
110 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
112 // Run with the full set of counters enabled: Profile=On
113 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
114 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
120 Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
122 // Save machinedata.json to /artifact/bin/ Jenkins dir
123 def archiveSettings = new ArchivalSettings()
124 archiveSettings.addFiles('Perf-*.xml')
125 archiveSettings.addFiles('Perf-*.etl')
126 archiveSettings.addFiles('Perf-*.log')
127 archiveSettings.addFiles('machinedata.json')
128 Utilities.addArchival(newJob, archiveSettings)
130 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
141 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
144 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests Correctness")
148 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
149 builder.triggerOnlyOnComment()
150 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
152 builder.triggerForBranch(branch)
153 builder.emitTrigger(newJob)
156 // Set a push trigger
157 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
158 builder.emitTrigger(newJob)
165 // Setup throughput perflab tests runs
166 [true, false].each { isPR ->
167 ['Windows_NT'].each { os ->
168 ['x64', 'x86'].each { arch ->
169 def architecture = arch
171 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
173 label('windows_clr_perf')
176 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
184 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
187 def configuration = 'Release'
188 def runType = isPR ? 'private' : 'rolling'
189 def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
194 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
195 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
196 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
197 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
198 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
199 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
200 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
201 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
202 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
203 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
204 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
205 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
206 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
207 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
208 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
212 // Save machinedata.json to /artifact/bin/ Jenkins dir
213 def archiveSettings = new ArchivalSettings()
214 archiveSettings.addFiles('throughput-*.csv')
215 Utilities.addArchival(newJob, archiveSettings)
217 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
220 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
221 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
222 builder.triggerOnlyOnComment()
223 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
224 builder.triggerForBranch(branch)
225 builder.emitTrigger(newJob)
228 // Set a push trigger
229 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
230 builder.emitTrigger(newJob)
236 def static getFullPerfJobName(def project, def os, def isPR) {
237 return Utilities.getFullJobName(project, "perf_${os}", isPR)
240 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
241 [true, false].each { isPR ->
242 def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
243 def architecture = 'x64'
244 def configuration = 'Release'
246 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
247 ['RHEL7.2'].each { os ->
248 def newBuildJob = job(fullBuildJobName) {
250 shell("./build.sh verbose ${architecture} ${configuration}")
253 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
254 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
255 Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
258 // Actual perf testing on the following OSes
259 def perfOSList = ['Ubuntu14.04']
260 perfOSList.each { os ->
261 def newJob = job(getFullPerfJobName(project, os, isPR)) {
263 label('linux_clr_perf')
266 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
274 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
279 // Cap the maximum number of iterations to 21.
280 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
281 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
282 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
285 def osGroup = getOSGroup(os)
286 def runType = isPR ? 'private' : 'rolling'
287 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
290 shell("./tests/scripts/perf-prep.sh")
291 shell("./init-tools.sh")
292 copyArtifacts(fullBuildJobName) {
293 includePatterns("bin/**")
295 buildNumber('\${PRODUCT_BUILD}')
298 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
299 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
300 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
301 shell("""./tests/scripts/run-xunit-perf.sh \\
302 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
303 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
304 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
305 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
306 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
307 --runType=\"${runType}\" \\
308 --benchViewOS=\"${os}\" \\
309 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
310 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
311 --uploadToBenchview""")
315 // Save machinedata.json to /artifact/bin/ Jenkins dir
316 def archiveSettings = new ArchivalSettings()
317 archiveSettings.addFiles('Perf-*.log')
318 archiveSettings.addFiles('Perf-*.xml')
319 archiveSettings.addFiles('machinedata.json')
320 Utilities.addArchival(newJob, archiveSettings)
322 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
324 // For perf, we need to keep the run results longer
326 // Enable the log rotator
328 artifactDaysToKeep(7)
330 artifactNumToKeep(25)
336 def flowJobPerfRunList = perfOSList.collect { os ->
337 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
339 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
342 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
346 // First, build the bits on RHEL7.2
347 b = build(params, '${fullBuildJobName}')
349 // Then, run the perf tests
351 ${flowJobPerfRunList.join(",\n ")}
356 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
357 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
360 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
361 builder.setGithubContext("Linux Perf Test Flow")
362 builder.triggerOnlyOnComment()
363 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
364 builder.triggerForBranch(branch)
365 builder.emitTrigger(newFlowJob)
368 // Set a push trigger
369 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
370 builder.emitTrigger(newFlowJob)
375 def static getFullThroughputJobName(def project, def os, def isPR) {
376 return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
379 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
380 [true, false].each { isPR ->
381 def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
382 def architecture = 'x64'
383 def configuration = 'Release'
385 // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
386 ['RHEL7.2'].each { os ->
387 def newBuildJob = job(fullBuildJobName) {
389 shell("./build.sh verbose ${architecture} ${configuration}")
392 Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
393 Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
394 Utilities.addArchival(newBuildJob, "bin/Product/**")
397 // Actual perf testing on the following OSes
398 def throughputOSList = ['Ubuntu14.04']
399 throughputOSList.each { os ->
400 def newJob = job(getFullThroughputJobName(project, os, isPR)) {
402 label('linux_clr_perf')
405 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
413 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
418 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
421 def osGroup = getOSGroup(os)
422 def runType = isPR ? 'private' : 'rolling'
423 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
426 shell("bash ./tests/scripts/perf-prep.sh --throughput")
427 shell("./init-tools.sh")
428 copyArtifacts(fullBuildJobName) {
429 includePatterns("bin/Product/**")
431 buildNumber('\${PRODUCT_BUILD}')
434 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
435 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
436 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
437 shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
438 -arch \"${architecture}\" \\
440 -configuration \"${configuration}\" \\
441 -clr_root \"\${WORKSPACE}\" \\
442 -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
443 -run_type \"${runType}\" \\
444 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
448 // Save machinedata.json to /artifact/bin/ Jenkins dir
449 def archiveSettings = new ArchivalSettings()
450 archiveSettings.addFiles('throughput-*.csv')
451 archiveSettings.addFiles('machinedata.json')
452 Utilities.addArchival(newJob, archiveSettings)
454 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
456 // For perf, we need to keep the run results longer
458 // Enable the log rotator
460 artifactDaysToKeep(7)
462 artifactNumToKeep(25)
468 def flowJobTPRunList = throughputOSList.collect { os ->
469 "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
471 def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
474 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
478 // First, build the bits on RHEL7.2
479 b = build(params, '${fullBuildJobName}')
481 // Then, run the perf tests
483 ${flowJobTPRunList.join(",\n ")}
488 Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
489 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
492 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
493 builder.setGithubContext("Linux Throughput Perf Test Flow")
494 builder.triggerOnlyOnComment()
495 builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
496 builder.triggerForBranch(branch)
497 builder.emitTrigger(newFlowJob)
500 // Set a push trigger
501 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
502 builder.emitTrigger(newFlowJob)
507 // Setup ILLink tests
508 [true, false].each { isPR ->
509 ['Windows_NT'].each { os ->
510 ['x64'].each { arch ->
511 def architecture = arch
512 def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}", isPR)) {
514 label('windows_clr_perf')
517 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
525 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
531 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
532 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
534 def configuration = 'Release'
535 def runType = isPR ? 'private' : 'rolling'
536 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
537 def uploadString = '-uploadToBenchview'
541 batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
542 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
543 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
545 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
546 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
547 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
548 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
549 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
550 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
551 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
552 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
553 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
555 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
557 // Run with just stopwatch: Profile=Off
558 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -nowarmup -runtype ${runType} -scenarioTest -group ILLink")
562 // Save machinedata.json to /artifact/bin/ Jenkins dir
563 def archiveSettings = new ArchivalSettings()
564 archiveSettings.addFiles('Perf-*.xml')
565 archiveSettings.addFiles('Perf-*.log')
566 archiveSettings.addFiles('machinedata.json')
567 Utilities.addArchival(newJob, archiveSettings)
569 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
580 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
581 builder.setGithubContext("${os} ${arch} ILLink Perf Tests")
582 builder.triggerOnlyOnComment()
583 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink\\W+perf.*")
584 builder.triggerForBranch(branch)
585 builder.emitTrigger(newJob)
588 // Set a push trigger
589 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
590 builder.emitTrigger(newJob)
596 Utilities.createHelperJob(this, project, branch,
597 "Welcome to the ${project} Perf help",