Merge pull request #15281 from sdmaclea/PR-ARM64-Fix-UNROLL_LIMIT
[platform/upstream/coreclr.git] / perf.groovy
1 // Import the utility functionality.
2
3 import jobs.generation.*;
4
5 def project = GithubProject
6 def branch = GithubBranchName
7 def projectName = Utilities.getFolderName(project)
8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
9
10 def static getOSGroup(def os) {
11     def osGroupMap = ['Ubuntu14.04':'Linux',
12         'RHEL7.2': 'Linux',
13         'Ubuntu16.04': 'Linux',
14         'Debian8.4':'Linux',
15         'Fedora24':'Linux',
16         'OSX':'OSX',
17         'Windows_NT':'Windows_NT',
18         'FreeBSD':'FreeBSD',
19         'CentOS7.1': 'Linux',
20         'OpenSUSE13.2': 'Linux',
21         'OpenSUSE42.1': 'Linux',
22         'LinuxARMEmulator': 'Linux']
23     def osGroup = osGroupMap.get(os, null)
24     assert osGroup != null : "Could not find os group for ${os}"
25     return osGroupMap[os]
26 }
27
28 // Setup perflab tests runs
29 [true, false].each { isPR ->
30     ['Windows_NT'].each { os ->
31         ['x64', 'x86'].each { arch ->
32             ['ryujit'].each { jit ->
33                 ['full_opt', 'min_opt'].each { opt_level ->
34
35                     def architecture = arch
36                     def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
37                     def testEnv = ""
38
39                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
40                         // Set the label.
41                         label('windows_server_2016_clr_perf')
42                         wrappers {
43                             credentialsBinding {
44                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
45                             }
46                         }
47
48                         if (isPR) {
49                             parameters {
50                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
51                             }
52                         }
53
54                         parameters {
55                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
56                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
57                         }
58
59                         def configuration = 'Release'
60                         def runType = isPR ? 'private' : 'rolling'
61                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
62                         def uploadString = '-uploadToBenchview'
63
64                         steps {
65                             // Batch
66
67                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
68                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
69                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
70                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
71                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
72                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
73                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
74                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
75                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
76                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
77                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
78                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
79
80                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
81
82                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
83
84                             // Run with just stopwatch: Profile=Off
85                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
86                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
87
88                             // Run with the full set of counters enabled: Profile=On
89                             if (opt_level != 'min_opt') {
90                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
91                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
92                             }
93                         }
94                     }
95
96                     def archiveSettings = new ArchivalSettings()
97                     archiveSettings.addFiles('bin/sandbox_logs/**')
98                     archiveSettings.addFiles('machinedata.json')
99
100                     Utilities.addArchival(newJob, archiveSettings)
101                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
102
103                     newJob.with {
104                         logRotator {
105                             artifactDaysToKeep(30)
106                             daysToKeep(30)
107                             artifactNumToKeep(200)
108                             numToKeep(200)
109                         }
110                         wrappers {
111                             timeout {
112                                 absolute(240)
113                             }
114                         }
115                     }
116
117                     if (isPR) {
118                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
119                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
120
121                         def opts = ""
122                         if (opt_level == 'min_opt') {
123                             opts = '\\W+min_opts'
124                         }
125                         def jitt = ""
126                         if (jit != 'ryujit') {
127                             jitt = "\\W+${jit}"
128                         }
129
130                         builder.triggerOnlyOnComment()
131                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
132                         builder.triggerForBranch(branch)
133                         builder.emitTrigger(newJob)
134                     }
135                     else {
136                         // Set a push trigger
137                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
138                         builder.emitTrigger(newJob)
139                     }
140                 }
141             }
142         }
143     }
144 }
145
146 // Setup throughput perflab tests runs
147 [true, false].each { isPR ->
148     ['Windows_NT'].each { os ->
149         ['x64', 'x86'].each { arch ->
150             ['ryujit'].each { jit ->
151                 [true, false].each { pgo_optimized ->
152                     ['full_opt', 'min_opt'].each { opt_level ->
153                         def architecture = arch
154
155                         pgo_build = ""
156                         pgo_test = ""
157                         pgo_string = "pgo"
158                         if (!pgo_optimized) {
159                             pgo_build = " -nopgooptimize"
160                             pgo_test = " -nopgo"
161                             pgo_string = "nopgo"
162                         }
163
164                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) {
165                             // Set the label.
166                             label('windows_server_2016_clr_perf')
167                             wrappers {
168                                 credentialsBinding {
169                                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
170                                 }
171                             }
172
173                             if (isPR) {
174                                 parameters {
175                                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
176                                 }
177                             }
178
179                             def configuration = 'Release'
180                             def runType = isPR ? 'private' : 'rolling'
181                             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
182
183                             steps {
184                                 // Batch
185                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
186                                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
187                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
188                                 batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
189                                 //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
190                                 //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
191                                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
192                                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
193                                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
194                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
195                                 "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
196                                 batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
197                                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
198                                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
199                                 batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
200                             }
201                         }
202
203                         // Save machinedata.json to /artifact/bin/ Jenkins dir
204                         def archiveSettings = new ArchivalSettings()
205                         archiveSettings.addFiles('throughput-*.csv')
206                         Utilities.addArchival(newJob, archiveSettings)
207
208                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
209
210                         if (isPR) {
211                             def opts = ""
212                             if (opt_level == 'min_opt') {
213                                 opts = '\\W+min_opts'
214                             }
215
216                             def jitt = ""
217                             if (jit != 'ryujit') {
218                                 jitt = "\\W+${jit}"
219                             }
220
221                             def pgo_trigger = ""
222                             if (pgo_optimized) {
223                                 pgo_trigger = "\\W+nopgo"
224                             }
225
226
227                             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
228                             builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests")
229                             builder.triggerOnlyOnComment()
230                             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*")
231                             builder.triggerForBranch(branch)
232                             builder.emitTrigger(newJob)
233                         }
234                         else {
235                             // Set a push trigger
236                             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
237                             builder.emitTrigger(newJob)
238                         }
239                     }
240                 }
241             }
242         }
243     }
244 }
245
246 def static getFullPerfJobName(def project, def os, def isPR) {
247     return Utilities.getFullJobName(project, "perf_${os}", isPR)
248 }
249
250 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
251 [true, false].each { isPR ->
252     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
253     def architecture = 'x64'
254     def configuration = 'Release'
255
256     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
257     ['RHEL7.2'].each { os ->
258         def newBuildJob = job(fullBuildJobName) {
259             steps {
260                 shell("./build.sh verbose ${architecture} ${configuration}")
261             }
262         }
263         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
264         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
265         Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
266     }
267
268
269     // Actual perf testing on the following OSes
270     def perfOSList = ['Ubuntu14.04']
271     perfOSList.each { os ->
272         def newJob = job(getFullPerfJobName(project, os, isPR)) {
273
274             label('ubuntu_1604_clr_perf')
275             wrappers {
276                 credentialsBinding {
277                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
278                 }
279             }
280
281             if (isPR) {
282                 parameters {
283                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
284                 }
285             }
286
287             parameters {
288                 // Cap the maximum number of iterations to 21.
289                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
290                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
291                 stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
292             }
293
294             def osGroup = getOSGroup(os)
295             def runType = isPR ? 'private' : 'rolling'
296             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
297
298             steps {
299                 shell("./tests/scripts/perf-prep.sh")
300                 shell("./init-tools.sh")
301                 copyArtifacts(fullBuildJobName) {
302                     includePatterns("bin/**")
303                     buildSelector {
304                         buildNumber('\${PRODUCT_BUILD}')
305                     }
306                 }
307                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
308                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
309                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
310                 shell("""./tests/scripts/run-xunit-perf.sh \\
311                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
312                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
313                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
314                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
315                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
316                 --runType=\"${runType}\" \\
317                 --benchViewOS=\"${os}\" \\
318                 --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
319                 --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
320                 --uploadToBenchview""")
321                 shell("mkdir -p bin/toArchive/sandbox/Logs/")
322                 shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
323             }
324         }
325
326         def archiveSettings = new ArchivalSettings()
327         archiveSettings.addFiles('bin/toArchive/**')
328         archiveSettings.addFiles('machinedata.json')
329
330         Utilities.addArchival(newJob, archiveSettings)
331         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
332
333         // For perf, we need to keep the run results longer
334         newJob.with {
335             // Enable the log rotator
336             logRotator {
337                 artifactDaysToKeep(30)
338                 daysToKeep(30)
339                 artifactNumToKeep(200)
340                 numToKeep(200)
341             }
342             wrappers {
343                 timeout {
344                     absolute(240)
345                 }
346             }
347         }
348     } // os
349
350     def flowJobPerfRunList = perfOSList.collect { os ->
351         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
352     }
353     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
354         if (isPR) {
355             parameters {
356                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
357             }
358         }
359         buildFlow("""
360 // First, build the bits on RHEL7.2
361 b = build(params, '${fullBuildJobName}')
362
363 // Then, run the perf tests
364 parallel(
365     ${flowJobPerfRunList.join(",\n    ")}
366 )
367 """)
368     }
369
370     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
371     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
372
373     if (isPR) {
374         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
375         builder.setGithubContext("Linux Perf Test Flow")
376         builder.triggerOnlyOnComment()
377         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
378         builder.triggerForBranch(branch)
379         builder.emitTrigger(newFlowJob)
380     }
381     else {
382         // Set a push trigger
383         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
384         builder.emitTrigger(newFlowJob)
385     }
386
387 } // isPR
388
389 def static getFullThroughputJobName(def project, def os, def isPR) {
390     return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
391 }
392
393 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
394 [true, false].each { isPR ->
395     def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
396     def architecture = 'x64'
397     def configuration = 'Release'
398
399     // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
400     ['RHEL7.2'].each { os ->
401         def newBuildJob = job(fullBuildJobName) {
402             steps {
403                 shell("./build.sh verbose ${architecture} ${configuration}")
404             }
405         }
406         Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
407         Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
408         Utilities.addArchival(newBuildJob, "bin/Product/**")
409     }
410
411     // Actual perf testing on the following OSes
412     def throughputOSList = ['Ubuntu14.04']
413     def throughputOptLevelList = ['full_opt', 'min_opt']
414
415     def throughputOSOptLevelList = []
416
417     throughputOSList.each { os ->
418         throughputOptLevelList.each { opt_level ->
419             throughputOSOptLevelList.add("${os}_${opt_level}")
420         }
421     }
422
423     throughputOSList.each { os ->
424         throughputOptLevelList.each { opt_level ->
425             def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
426
427                 label('ubuntu_1604_clr_perf')
428                     wrappers {
429                         credentialsBinding {
430                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
431                         }
432                     }
433
434                 if (isPR) {
435                     parameters {
436                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
437                     }
438                 }
439
440                 parameters {
441                     stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
442                 }
443
444                 def osGroup = getOSGroup(os)
445                 def runType = isPR ? 'private' : 'rolling'
446                 def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
447
448                 steps {
449                     shell("bash ./tests/scripts/perf-prep.sh --throughput")
450                     shell("./init-tools.sh")
451                     copyArtifacts(fullBuildJobName) {
452                         includePatterns("bin/Product/**")
453                         buildSelector {
454                             buildNumber('\${PRODUCT_BUILD}')
455                         }
456                     }
457                     shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
458                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
459                     "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
460                     shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
461                     -arch \"${architecture}\" \\
462                     -os \"${os}\" \\
463                     -configuration \"${configuration}\" \\
464                     -opt_level \"${opt_level}\" \\
465                     -clr_root \"\${WORKSPACE}\" \\
466                     -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
467                     -run_type \"${runType}\" \\
468                     -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
469                 }
470             }
471
472             // Save machinedata.json to /artifact/bin/ Jenkins dir
473             def archiveSettings = new ArchivalSettings()
474             archiveSettings.addFiles('throughput-*.csv')
475             archiveSettings.addFiles('machinedata.json')
476             Utilities.addArchival(newJob, archiveSettings)
477
478             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
479
480             // For perf, we need to keep the run results longer
481             newJob.with {
482                 // Enable the log rotator
483                 logRotator {
484                     artifactDaysToKeep(7)
485                     daysToKeep(300)
486                     artifactNumToKeep(25)
487                     numToKeep(1000)
488                 }
489             }
490         } // opt_level
491     } // os
492
493     def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
494         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
495     }
496     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
497         if (isPR) {
498             parameters {
499                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
500             }
501         }
502         buildFlow("""
503 // First, build the bits on RHEL7.2
504 b = build(params, '${fullBuildJobName}')
505
506 // Then, run the perf tests
507 parallel(
508     ${flowJobTPRunList.join(",\n    ")}
509 )
510 """)
511     }
512
513     Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
514     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
515
516     if (isPR) {
517         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
518         builder.setGithubContext("Linux Throughput Perf Test Flow")
519         builder.triggerOnlyOnComment()
520         builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
521         builder.triggerForBranch(branch)
522         builder.emitTrigger(newFlowJob)
523     }
524     else {
525         // Set a push trigger
526         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
527         builder.emitTrigger(newFlowJob)
528     }
529
530 } // isPR
531
532 // Setup CoreCLR-Scenarios tests
533 [true, false].each { isPR ->
534     ['Windows_NT'].each { os ->
535         ['x64', 'x86'].each { arch ->
536             ['ryujit'].each { jit ->
537                 ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
538                     def architecture = arch
539                     def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
540
541                         def testEnv = ""
542
543                         // Set the label.
544                         label('windows_server_2016_clr_perf')
545                         wrappers {
546                             credentialsBinding {
547                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
548                             }
549                         }
550
551                         if (isPR) {
552                             parameters {
553                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
554                             }
555                         }
556
557                         parameters {
558                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
559                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
560                         }
561
562                         def configuration = 'Release'
563                         def runType = isPR ? 'private' : 'rolling'
564                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
565                         def uploadString = '-uploadToBenchview'
566
567                         steps {
568                             // Batch
569                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
570                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
571                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
572
573                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
574                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
575                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
576                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
577                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
578                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
579                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
580                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
581                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
582
583                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
584
585                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
586
587                             // Profile=Off
588                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
589
590                             // Profile=On
591                             if (opt_level != 'min_opt') {
592                                 batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
593                             }
594                         }
595                     }
596
597                     def archiveSettings = new ArchivalSettings()
598                     archiveSettings.addFiles('bin/sandbox_logs/**')
599                     archiveSettings.addFiles('machinedata.json')
600
601                     Utilities.addArchival(newJob, archiveSettings)
602                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
603
604                     newJob.with {
605                         logRotator {
606                             artifactDaysToKeep(30)
607                             daysToKeep(30)
608                             artifactNumToKeep(200)
609                             numToKeep(200)
610                         }
611                         wrappers {
612                             timeout {
613                                 absolute(240)
614                             }
615                         }
616                     }
617
618                     if (isPR) {
619                         def opts = ""
620                         if (opt_level == 'min_opt') {
621                             opts = '\\W+min_opts'
622                         }
623                         def jitt = ""
624                         if (jit != 'ryujit') {
625                             jitt = "\\W+${jit}"
626                         }
627
628                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
629                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
630                         builder.triggerOnlyOnComment()
631                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
632                         builder.triggerForBranch(branch)
633                         builder.emitTrigger(newJob)
634                     }
635                     else {
636                         // Set a push trigger
637                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
638                         builder.emitTrigger(newJob)
639                     }
640                 }
641             }
642         }
643     }
644 }
645
646 // Setup size-on-disk test
647 ['Windows_NT'].each { os ->
648     ['x64', 'x86'].each { arch ->
649         def architecture = arch
650         def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
651
652             wrappers {
653                 credentialsBinding {
654                     string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
655                 }
656             }
657
658             def channel = 'master'
659             def configuration = 'Release'
660             def runType = 'rolling'
661             def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
662             def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
663             def coreRoot = "${testBin}\\Tests\\Core_Root"
664             def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
665
666             steps {
667                 // Install nuget and get BenchView tools
668                 batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
669                 batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
670                 batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
671
672                 // Generate submission metadata for BenchView
673                 // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
674                 // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
675                 batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
676                 "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
677                 "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
678                 "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
679                 "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
680
681                 // Generate machine data from BenchView
682                 batchFile("py \"${benchViewTools}\\machinedata.py\"")
683
684                 // Build CoreCLR and gnerate test layout
685                 batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
686                 batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
687
688                 // Run the size on disk benchmark
689                 batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
690
691                 // From sodbench.csv, create measurment.json, then submission.json
692                 batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
693                 batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
694
695                 // If this is a PR, upload submission.json
696                 batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
697             }
698         }
699
700         Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
701
702         def archiveSettings = new ArchivalSettings()
703         archiveSettings.addFiles('bin/toArchive/**')
704         archiveSettings.addFiles('machinedata.json')
705
706         Utilities.addArchival(newJob, archiveSettings)
707         Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
708
709         // Set the cron job here.  We run nightly on each flavor, regardless of code changes
710         Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
711
712         newJob.with {
713             logRotator {
714                 artifactDaysToKeep(30)
715                 daysToKeep(30)
716                 artifactNumToKeep(200)
717                 numToKeep(200)
718             }
719             wrappers {
720                 timeout {
721                     absolute(240)
722                 }
723             }
724         }
725     }
726 }
727
728 // Setup IlLink tests
729 [true, false].each { isPR ->
730     ['Windows_NT'].each { os ->
731         ['x64'].each { arch ->
732             ['ryujit'].each { jit ->
733                 ['full_opt'].each { opt_level ->
734                     def architecture = arch
735                     def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {
736
737                         def testEnv = ""
738                         wrappers {
739                             credentialsBinding {
740                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
741                             }
742                         }
743
744                         if (isPR) {
745                             parameters {
746                                 stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
747                             }
748                         }
749
750                         parameters {
751                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
752                             stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
753                         }
754
755                         def configuration = 'Release'
756                         def runType = isPR ? 'private' : 'rolling'
757                         def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
758                         def uploadString = '-uploadToBenchview'
759
760                         steps {
761                             // Batch
762                             batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
763                             batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
764                             batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
765
766                             //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
767                             //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
768                             batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
769                             "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
770                             "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
771                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
772                             "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
773                             batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
774                             batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
775
776                             batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
777
778                             def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
779
780                             // Scenario: ILLink
781                             batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
782                         }
783                     }
784
785                     def archiveSettings = new ArchivalSettings()
786                     archiveSettings.addFiles('bin/sandbox_logs/**')
787                     archiveSettings.addFiles('machinedata.json')
788
789                     // Set the label (currently we are only measuring size, therefore we are running on VM).
790                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
791                     Utilities.addArchival(newJob, archiveSettings)
792                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
793
794                     newJob.with {
795                         logRotator {
796                             artifactDaysToKeep(30)
797                             daysToKeep(30)
798                             artifactNumToKeep(200)
799                             numToKeep(200)
800                         }
801                         wrappers {
802                             timeout {
803                                 absolute(240)
804                             }
805                         }
806                     }
807
808                     if (isPR) {
809                         TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
810                         builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
811                         builder.triggerOnlyOnComment()
812                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
813                         builder.triggerForBranch(branch)
814                         builder.emitTrigger(newJob)
815                     }
816                     else {
817                         // Set a push trigger
818                         TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
819                         builder.emitTrigger(newJob)
820                     }
821                 }
822             }
823         }
824     }
825 }
826
827 Utilities.createHelperJob(this, project, branch,
828     "Welcome to the ${project} Perf help",
829     "Have a nice day!")