Merge pull request #15332 from sdmaclea/PR-ARM64-SIMDIntrinsicGetItem-Contain-Memop
[platform/upstream/coreclr.git] / perf.groovy
1 // Import the utility functionality.
2
3 import jobs.generation.*;
4
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
9
10 def static getOSGroup(def os) {
11     def osGroupMap = ['Ubuntu14.04':'Linux',
12         'RHEL7.2': 'Linux',
13         'Ubuntu16.04': 'Linux',
14         'Debian8.4':'Linux',
15         'Fedora24':'Linux',
16         'OSX':'OSX',
17         'Windows_NT':'Windows_NT',
18         'FreeBSD':'FreeBSD',
19         'CentOS7.1': 'Linux',
20         'OpenSUSE13.2': 'Linux',
21         'OpenSUSE42.1': 'Linux',
22         'LinuxARMEmulator': 'Linux']
23     def osGroup = osGroupMap.get(os, null)
24     assert osGroup != null : "Could not find os group for ${os}"
25     return osGroupMap[os]
26 }
27
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30     ['Windows_NT'].each { os ->
31         ['x64', 'x86'].each { arch ->
32             ['ryujit'].each { jit ->
33                 ['full_opt', 'min_opt'].each { opt_level ->
34
35                     def architecture = arch
36                     def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
37                     def testEnv = ""
38
39                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
40                         // Set the label.
41                         label('windows_server_2016_clr_perf')
42                         wrappers {
43                             credentialsBinding {
44                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
45                             }
46                         }
47
48                         if (isPR) {
49                             parameters {
50                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
51                             }
52                         }
53
54                         parameters {
55                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
56                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
57                         }
58
59                         def configuration = 'Release'
60                         def runType = isPR ? 'private' : 'rolling'
61                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
62                         def uploadString = '-uploadToBenchview'
63
64                         steps {
65                             // Batch
66
67                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
68                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
69                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
70                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
71                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
72                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
73                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
74                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
75                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
76                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
77                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
78                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
79
80                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
81
82                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
83
84                             // Run with just stopwatch: Profile=Off
85                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
86                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
87
88                             // Run with the full set of counters enabled: Profile=On
89                             if (opt_level != 'min_opt') {
90                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
91                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
92                             }
93                         }
94                     }
95
96                     def archiveSettings = new ArchivalSettings()
97                     archiveSettings.addFiles('bin/sandbox_logs/**')
98                     archiveSettings.addFiles('machinedata.json')
99
100                     Utilities.addArchival(newJob, archiveSettings)
101                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
102
103                     newJob.with {
104                         logRotator {
105                             artifactDaysToKeep(30)
106                             daysToKeep(30)
107                             artifactNumToKeep(200)
108                             numToKeep(200)
109                         }
110                         wrappers {
111                             timeout {
112                                 absolute(240)
113                             }
114                         }
115                     }
116
117                     if (isPR) {
118                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
119                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
120
121                         def opts = ""
122                         if (opt_level == 'min_opt') {
123                             opts = '\\W+min_opts'
124                         }
125                         def jitt = ""
126                         if (jit != 'ryujit') {
127                             jitt = "\\W+${jit}"
128                         }
129
130                         builder.triggerOnlyOnComment()
131                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
132                         builder.triggerForBranch(branch)
133                         builder.emitTrigger(newJob)
134                     }
135                     else if (opt_level == 'full_opt') {
136                         // Set a push trigger
137                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
138                         builder.emitTrigger(newJob)
139                     }
140                     else {
141                         // Set periodic trigger
142                         Utilities.addPeriodicTrigger(newJob, '@daily')
143                     }
144                 }
145             }
146         }
147     }
148 }
149
150 // Setup throughput perflab tests runs
151 [true, false].each { isPR ->
152     ['Windows_NT'].each { os ->
153         ['x64', 'x86'].each { arch ->
154             ['ryujit'].each { jit ->
155                 [true, false].each { pgo_optimized ->
156                     ['full_opt', 'min_opt'].each { opt_level ->
157                         def architecture = arch
158
159                         pgo_build = ""
160                         pgo_test = ""
161                         pgo_string = "pgo"
162                         if (!pgo_optimized) {
163                             pgo_build = " -nopgooptimize"
164                             pgo_test = " -nopgo"
165                             pgo_string = "nopgo"
166                         }
167
168                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
169                             // Set the label.
170                             label('windows_server_2016_clr_perf')
171                             wrappers {
172                                 credentialsBinding {
173                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
174                                 }
175                             }
176
177                             if (isPR) {
178                                 parameters {
179                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
180                                 }
181                             }
182
183                             def configuration = 'Release'
184                             def runType = isPR ? 'private' : 'rolling'
185                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
186
187                             steps {
188                                 // Batch
189                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
190                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
191                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
192                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
193                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
194                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
195                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
196                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
197                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
198                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
199                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
200                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
201                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
202                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
203                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
204                             }
205                         }
206
207                         // Save machinedata.json to /artifact/bin/ Jenkins dir
208                         def archiveSettings = new ArchivalSettings()
209                         archiveSettings.addFiles('throughput-*.csv')
210                         Utilities.addArchival(newJob, archiveSettings)
211
212                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
213
214                         if (isPR) {
215                             def opts = ""
216                             if (opt_level == 'min_opt') {
217                                 opts = '\\W+min_opts'
218                             }
219
220                             def jitt = ""
221                             if (jit != 'ryujit') {
222                                 jitt = "\\W+${jit}"
223                             }
224
225                             def pgo_trigger = ""
226                             if (pgo_optimized) {
227                                 pgo_trigger = "\\W+nopgo"
228                             }
229
230
231                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
232                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
233                             builder.triggerOnlyOnComment()
234                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
235                             builder.triggerForBranch(branch)
236                             builder.emitTrigger(newJob)
237                         }
238                         else if (opt_level == 'full_opt' && pgo_optimized) {
239                             // Set a push trigger
240                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
241                             builder.emitTrigger(newJob)
242                         }
243                         else {
244                             // Set periodic trigger
245                             Utilities.addPeriodicTrigger(newJob, '@daily')
246                         }
247                     }
248                 }
249             }
250         }
251     }
252 }
253
254 def static getFullPerfJobName(def project, def os, def isPR) {
255     return Utilities.getFullJobName(project, "perf_${os}", isPR)
256 }
257
258 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
259 [true, false].each { isPR ->
260     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
261     def architecture = 'x64'
262     def configuration = 'Release'
263
264     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
265     ['RHEL7.2'].each { os ->
266         def newBuildJob = job(fullBuildJobName) {
267             steps {
268                 shell("./build.sh verbose ${architecture} ${configuration}")
269             }
270         }
271         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
272         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
273         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
274     }
275
276
277     // Actual perf testing on the following OSes
278     def perfOSList = ['Ubuntu16.04']
279     perfOSList.each { os ->
280         def newJob = job(getFullPerfJobName(project, os, isPR)) {
281
282             label('ubuntu_1604_clr_perf')
283             wrappers {
284                 credentialsBinding {
285                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
286                 }
287             }
288
289             if (isPR) {
290                 parameters {
291                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
292                 }
293             }
294
295             parameters {
296                 // Cap the maximum number of iterations to 21.
297                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
298                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
299                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
300             }
301
302             def osGroup = getOSGroup(os)
303             def runType = isPR ? 'private' : 'rolling'
304             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
305
306             steps {
307                 shell("./tests/scripts/perf-prep.sh")
308                 shell("./init-tools.sh")
309                 copyArtifacts(fullBuildJobName) {
310                     includePatterns("bin/**")
311                     buildSelector {
312                         buildNumber('\${PRODUCT_BUILD}')
313                     }
314                 }
315                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
316                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
317                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
318                 shell("""./tests/scripts/run-xunit-perf.sh \\
319                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
320                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
321                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
322                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
323                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
324                 --runType=\"${runType}\" \\
325                 --benchViewOS=\"${os}\" \\
326                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
327                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
328                 --uploadToBenchview""")
329                 shell("mkdir -p bin/toArchive/sandbox/Logs/")
330                 shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
331             }
332         }
333
334         def archiveSettings = new ArchivalSettings()
335         archiveSettings.addFiles('bin/toArchive/**')
336         archiveSettings.addFiles('machinedata.json')
337
338         Utilities.addArchival(newJob, archiveSettings)
339         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
340
341         // For perf, we need to keep the run results longer
342         newJob.with {
343             // Enable the log rotator
344             logRotator {
345                 artifactDaysToKeep(30)
346                 daysToKeep(30)
347                 artifactNumToKeep(200)
348                 numToKeep(200)
349             }
350             wrappers {
351                 timeout {
352                     absolute(240)
353                 }
354             }
355         }
356     } // os
357
358     def flowJobPerfRunList = perfOSList.collect { os ->
359         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
360     }
361     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
362         if (isPR) {
363             parameters {
364                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
365             }
366         }
367         buildFlow("""
368 // First, build the bits on RHEL7.2
369 b = build(params, '${fullBuildJobName}')
370
371 // Then, run the perf tests
372 parallel(
373     ${flowJobPerfRunList.join(",\n    ")}
374 )
375 """)
376     }
377
378     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
379     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
380
381     if (isPR) {
382         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
383         builder.setGithubContext("Linux Perf Test Flow")
384         builder.triggerOnlyOnComment()
385         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
386         builder.triggerForBranch(branch)
387         builder.emitTrigger(newFlowJob)
388     }
389     else {
390         // Set a push trigger
391         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
392         builder.emitTrigger(newFlowJob)
393     }
394
395 } // isPR
396
397 def static getFullThroughputJobName(def project, def os, def isPR) {
398     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
399 }
400
401 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
402 [true, false].each { isPR ->
403     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
404     def architecture = 'x64'
405     def configuration = 'Release'
406
407     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
408     ['RHEL7.2'].each { os ->
409         def newBuildJob = job(fullBuildJobName) {
410             steps {
411                 shell("./build.sh verbose ${architecture} ${configuration}")
412             }
413         }
414         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
415         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
416         Utilities.addArchival(newBuildJob, "bin/Product/**")
417     }
418
419     // Actual perf testing on the following OSes
420     def throughputOSList = ['Ubuntu14.04']
421     def throughputOptLevelList = ['full_opt', 'min_opt']
422
423     def throughputOSOptLevelList = []
424
425     throughputOSList.each { os ->
426         throughputOptLevelList.each { opt_level ->
427             throughputOSOptLevelList.add("${os}_${opt_level}")
428         }
429     }
430
431     throughputOSList.each { os ->
432         throughputOptLevelList.each { opt_level ->
433             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
434
435                 label('ubuntu_1604_clr_perf')
436                     wrappers {
437                         credentialsBinding {
438                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
439                         }
440                     }
441
442                 if (isPR) {
443                     parameters {
444                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
445                     }
446                 }
447
448                 parameters {
449                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
450                 }
451
452                 def osGroup = getOSGroup(os)
453                 def runType = isPR ? 'private' : 'rolling'
454                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
455
456                 steps {
457                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
458                     shell("./init-tools.sh")
459                     copyArtifacts(fullBuildJobName) {
460                         includePatterns("bin/Product/**")
461                         buildSelector {
462                             buildNumber('\${PRODUCT_BUILD}')
463                         }
464                     }
465                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
466                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
467                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
468                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
469                     -arch \"${architecture}\" \\
470                     -os \"${os}\" \\
471                     -configuration \"${configuration}\" \\
472                     -opt_level \"${opt_level}\" \\
473                     -clr_root \"\${WORKSPACE}\" \\
474                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
475                     -run_type \"${runType}\" \\
476                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
477                 }
478             }
479
480             // Save machinedata.json to /artifact/bin/ Jenkins dir
481             def archiveSettings = new ArchivalSettings()
482             archiveSettings.addFiles('throughput-*.csv')
483             archiveSettings.addFiles('machinedata.json')
484             Utilities.addArchival(newJob, archiveSettings)
485
486             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
487
488             // For perf, we need to keep the run results longer
489             newJob.with {
490                 // Enable the log rotator
491                 logRotator {
492                     artifactDaysToKeep(7)
493                     daysToKeep(300)
494                     artifactNumToKeep(25)
495                     numToKeep(1000)
496                 }
497             }
498         } // opt_level
499     } // os
500
501     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
502         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
503     }
504     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
505         if (isPR) {
506             parameters {
507                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
508             }
509         }
510         buildFlow("""
511 // First, build the bits on RHEL7.2
512 b = build(params, '${fullBuildJobName}')
513
514 // Then, run the perf tests
515 parallel(
516     ${flowJobTPRunList.join(",\n    ")}
517 )
518 """)
519     }
520
521     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
522     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
523
524     if (isPR) {
525         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
526         builder.setGithubContext("Linux Throughput Perf Test Flow")
527         builder.triggerOnlyOnComment()
528         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
529         builder.triggerForBranch(branch)
530         builder.emitTrigger(newFlowJob)
531     }
532     else {
533         // Set a push trigger
534         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
535         builder.emitTrigger(newFlowJob)
536     }
537
538 } // isPR
539
540 // Setup CoreCLR-Scenarios tests
541 [true, false].each { isPR ->
542     ['Windows_NT'].each { os ->
543         ['x64', 'x86'].each { arch ->
544             ['ryujit'].each { jit ->
545                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
546                     def architecture = arch
547                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
548
549                         def testEnv = ""
550
551                         // Set the label.
552                         label('windows_server_2016_clr_perf')
553                         wrappers {
554                             credentialsBinding {
555                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
556                             }
557                         }
558
559                         if (isPR) {
560                             parameters {
561                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
562                             }
563                         }
564
565                         parameters {
566                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
567                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
568                         }
569
570                         def configuration = 'Release'
571                         def runType = isPR ? 'private' : 'rolling'
572                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
573                         def uploadString = '-uploadToBenchview'
574
575                         steps {
576                             // Batch
577                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
578                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
579                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
580
581                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
582                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
583                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
584                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
585                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
586                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
587                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
588                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
589                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
590
591                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
592
593                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
594
595                             // Profile=Off
596                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
597
598                             // Profile=On
599                             if (opt_level != 'min_opt') {
600                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
601                             }
602                         }
603                     }
604
605                     def archiveSettings = new ArchivalSettings()
606                     archiveSettings.addFiles('bin/sandbox_logs/**')
607                     archiveSettings.addFiles('machinedata.json')
608
609                     Utilities.addArchival(newJob, archiveSettings)
610                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
611
612                     newJob.with {
613                         logRotator {
614                             artifactDaysToKeep(30)
615                             daysToKeep(30)
616                             artifactNumToKeep(200)
617                             numToKeep(200)
618                         }
619                         wrappers {
620                             timeout {
621                                 absolute(240)
622                             }
623                         }
624                     }
625
626                     if (isPR) {
627                         def opts = ""
628                         if (opt_level == 'min_opt') {
629                             opts = '\\W+min_opts'
630                         }
631                         def jitt = ""
632                         if (jit != 'ryujit') {
633                             jitt = "\\W+${jit}"
634                         }
635
636                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
637                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
638                         builder.triggerOnlyOnComment()
639                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
640                         builder.triggerForBranch(branch)
641                         builder.emitTrigger(newJob)
642                     }
643                     else if (opt_level == 'full_opt') {
644                         // Set a push trigger
645                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
646                         builder.emitTrigger(newJob)
647                     }
648                     else {
649                         // Set periodic trigger
650                         Utilities.addPeriodicTrigger(newJob, '@daily')
651                     }
652                 }
653             }
654         }
655     }
656 }
657
658 // Setup size-on-disk test
659 ['Windows_NT'].each { os ->
660     ['x64', 'x86'].each { arch ->
661         def architecture = arch
662         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
663
664             wrappers {
665                 credentialsBinding {
666                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
667                 }
668             }
669
670             def channel = 'master'
671             def configuration = 'Release'
672             def runType = 'rolling'
673             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
674             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
675             def coreRoot = "${testBin}\\Tests\\Core_Root"
676             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
677
678             steps {
679                 // Install nuget and get BenchView tools
680                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
681                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
682                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
683
684                 // Generate submission metadata for BenchView
685                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
686                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
687                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
688                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
689                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
690                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
691                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
692
693                 // Generate machine data from BenchView
694                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
695
696                 // Build CoreCLR and gnerate test layout
697                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
698                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
699
700                 // Run the size on disk benchmark
701                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
702
703                 // From sodbench.csv, create measurment.json, then submission.json
704                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
705                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
706
707                 // If this is a PR, upload submission.json
708                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
709             }
710         }
711
712         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
713
714         def archiveSettings = new ArchivalSettings()
715         archiveSettings.addFiles('bin/toArchive/**')
716         archiveSettings.addFiles('machinedata.json')
717
718         Utilities.addArchival(newJob, archiveSettings)
719         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
720
721         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
722         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
723
724         newJob.with {
725             logRotator {
726                 artifactDaysToKeep(30)
727                 daysToKeep(30)
728                 artifactNumToKeep(200)
729                 numToKeep(200)
730             }
731             wrappers {
732                 timeout {
733                     absolute(240)
734                 }
735             }
736         }
737     }
738 }
739
740 // Setup IlLink tests
741 [true, false].each { isPR ->
742     ['Windows_NT'].each { os ->
743         ['x64'].each { arch ->
744             ['ryujit'].each { jit ->
745                 ['full_opt'].each { opt_level ->
746                     def architecture = arch
747                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
748
749                         def testEnv = ""
750                         wrappers {
751                             credentialsBinding {
752                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
753                             }
754                         }
755
756                         if (isPR) {
757                             parameters {
758                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
759                             }
760                         }
761
762                         parameters {
763                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
764                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
765                         }
766
767                         def configuration = 'Release'
768                         def runType = isPR ? 'private' : 'rolling'
769                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
770                         def uploadString = '-uploadToBenchview'
771
772                         steps {
773                             // Batch
774                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
775                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
776                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
777
778                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
779                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
780                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
781                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
782                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
783                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
784                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
785                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
786                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
787
788                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
789
790                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
791
792                             // Scenario: ILLink
793                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
794                         }
795                     }
796
797                     def archiveSettings = new ArchivalSettings()
798                     archiveSettings.addFiles('bin/sandbox_logs/**')
799                     archiveSettings.addFiles('machinedata.json')
800
801                     // Set the label (currently we are only measuring size, therefore we are running on VM).
802                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
803                     Utilities.addArchival(newJob, archiveSettings)
804                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
805
806                     newJob.with {
807                         logRotator {
808                             artifactDaysToKeep(30)
809                             daysToKeep(30)
810                             artifactNumToKeep(200)
811                             numToKeep(200)
812                         }
813                         wrappers {
814                             timeout {
815                                 absolute(240)
816                             }
817                         }
818                     }
819
820                     if (isPR) {
821                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
822                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
823                         builder.triggerOnlyOnComment()
824                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
825                         builder.triggerForBranch(branch)
826                         builder.emitTrigger(newJob)
827                     }
828                     else {
829                         // Set a push trigger
830                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
831                         builder.emitTrigger(newJob)
832                     }
833                 }
834             }
835         }
836     }
837 }
838
839 Utilities.createHelperJob(this, project, branch,
840     "Welcome to the ${project} Perf help",
841     "Have a nice day!")