1 // Import the utility functionality.
3 import jobs.generation.*;
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
10 def static getOSGroup(def os) {
11 def osGroupMap = ['Ubuntu14.04':'Linux',
13 'Ubuntu16.04': 'Linux',
17 'Windows_NT':'Windows_NT',
20 'OpenSUSE13.2': 'Linux',
21 'OpenSUSE42.1': 'Linux',
22 'LinuxARMEmulator': 'Linux']
23 def osGroup = osGroupMap.get(os, null)
24 assert osGroup != null : "Could not find os group for ${os}"
27 // Setup perflab tests runs
28 [true, false].each { isPR ->
29 ['Windows_NT'].each { os ->
30 ['x64', 'x86', 'x86jit32'].each { arch ->
31 def architecture = arch
34 if (arch == 'x86jit32')
37 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
39 else if (arch == 'x86')
41 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
44 def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}_${arch}", isPR)) {
46 label('windows_clr_perf')
49 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
57 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
60 def configuration = 'Release'
61 def runType = isPR ? 'private' : 'rolling'
62 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
67 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
68 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
69 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
70 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
71 batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
72 "set BENCHVIEWNAME=${benchViewName}\n" +
73 "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
74 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
75 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
76 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
77 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
79 if (arch == 'x86jit32')
81 // Download package and copy compatjit into Core_Root
82 batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
83 "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
86 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
88 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
89 batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
93 // Save machinedata.json to /artifact/bin/ Jenkins dir
94 def archiveSettings = new ArchivalSettings()
95 archiveSettings.addFiles('Perf-*.xml')
96 archiveSettings.addFiles('Perf-*.etl')
97 Utilities.addArchival(newJob, archiveSettings)
99 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
110 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
111 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
112 builder.triggerOnlyOnComment()
113 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
114 builder.triggerForBranch(branch)
115 builder.emitTrigger(newJob)
118 // Set a push trigger
119 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
120 builder.emitTrigger(newJob)
126 // Setup throughput perflab tests runs
127 [true, false].each { isPR ->
128 ['Windows_NT'].each { os ->
129 ['x64', 'x86', 'x86jit32'].each { arch ->
130 def architecture = arch
132 if (arch == 'x86jit32')
137 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
139 label('windows_clr_perf')
142 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
150 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
153 def configuration = 'Release'
154 def runType = isPR ? 'private' : 'rolling'
155 def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
160 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
161 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
162 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
163 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
164 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
165 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
166 batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
167 "set BENCHVIEWNAME=${benchViewName}\n" +
168 "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
169 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
170 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
171 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
172 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
173 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
174 if (arch == 'x86jit32')
176 // Download package and copy compatjit into Core_Root
177 batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
178 "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
180 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
184 // Save machinedata.json to /artifact/bin/ Jenkins dir
185 def archiveSettings = new ArchivalSettings()
186 archiveSettings.addFiles('throughput-*.csv')
187 Utilities.addArchival(newJob, archiveSettings)
189 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
192 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
193 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
194 builder.triggerOnlyOnComment()
195 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
196 builder.triggerForBranch(branch)
197 builder.emitTrigger(newJob)
200 // Set a push trigger
201 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
202 builder.emitTrigger(newJob)
208 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
209 [true, false].each { isPR ->
210 ['Ubuntu14.04'].each { os ->
211 def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
213 label('linux_clr_perf')
216 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
224 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
227 def osGroup = getOSGroup(os)
228 def architecture = 'x64'
229 def configuration = 'Release'
230 def runType = isPR ? 'private' : 'rolling'
231 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
234 shell("bash ./tests/scripts/perf-prep.sh")
235 shell("./init-tools.sh")
236 shell("./build.sh ${architecture} ${configuration}")
237 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
238 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
239 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
240 shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
241 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
242 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
243 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
244 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
245 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
246 --runType=\"${runType}\" \\
247 --benchViewOS=\"${os}\" \\
248 --uploadToBenchview""")
252 // Save machinedata.json to /artifact/bin/ Jenkins dir
253 def archiveSettings = new ArchivalSettings()
254 archiveSettings.addFiles('sandbox/perf-*.xml')
255 archiveSettings.addFiles('machinedata.json')
256 Utilities.addArchival(newJob, archiveSettings)
258 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
260 // For perf, we need to keep the run results longer
262 // Enable the log rotator
264 artifactDaysToKeep(7)
266 artifactNumToKeep(25)
271 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
272 builder.setGithubContext("${os} Perf Tests")
273 builder.triggerOnlyOnComment()
274 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
275 builder.triggerForBranch(branch)
276 builder.emitTrigger(newJob)
279 // Set a push trigger
280 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
281 builder.emitTrigger(newJob)
286 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
287 [true, false].each { isPR ->
288 ['Ubuntu14.04'].each { os ->
289 def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
291 label('linux_clr_perf')
294 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
302 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
305 def osGroup = getOSGroup(os)
306 def architecture = 'x64'
307 def configuration = 'Release'
308 def runType = isPR ? 'private' : 'rolling'
309 def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
312 shell("bash ./tests/scripts/perf-prep.sh --throughput")
313 shell("./init-tools.sh")
314 shell("./build.sh ${architecture} ${configuration}")
315 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
316 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
317 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
318 shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
319 -arch \"${architecture}\" \\
321 -configuration \"${configuration}\" \\
322 -clr_root \"\${WORKSPACE}\" \\
323 -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
324 -run_type \"${runType}\" \\
325 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
329 // Save machinedata.json to /artifact/bin/ Jenkins dir
330 def archiveSettings = new ArchivalSettings()
331 archiveSettings.addFiles('throughput-*.csv')
332 archiveSettings.addFiles('machinedata.json')
333 Utilities.addArchival(newJob, archiveSettings)
335 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
337 // For perf, we need to keep the run results longer
339 // Enable the log rotator
341 artifactDaysToKeep(7)
343 artifactNumToKeep(25)
348 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
349 builder.setGithubContext("${os} Throughput Perf Tests")
350 builder.triggerOnlyOnComment()
351 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
352 builder.triggerForBranch(branch)
353 builder.emitTrigger(newJob)
356 // Set a push trigger
357 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
358 builder.emitTrigger(newJob)