['Windows_NT'].each { os ->
['x64', 'x86'].each { arch ->
['ryujit'].each { jit ->
- if (arch == 'x64' && jit == 'legacy_backend') {
- return
- }
-
['full_opt', 'min_opt'].each { opt_level ->
def architecture = arch
def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
def testEnv = ""
- if (jit == 'legacy_backend') {
- testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
- }
-
def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
// Set the label.
label('windows_server_2016_clr_perf')
steps {
// Batch
- batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
- def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
+ def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
// Run with just stopwatch: Profile=Off
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\Off\\")
-
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\")
+ batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
+ batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
// Run with the full set of counters enabled: Profile=On
if (opt_level != 'min_opt') {
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
-
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
+ batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
+ batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
}
}
}
def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('bin/toArchive/**')
+ archiveSettings.addFiles('bin/sandbox_logs/**')
archiveSettings.addFiles('machinedata.json')
+ archiveSettings.setAlwaysArchive()
Utilities.addArchival(newJob, archiveSettings)
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
builder.triggerForBranch(branch)
builder.emitTrigger(newJob)
}
- else {
+ else if (opt_level == 'full_opt') {
// Set a push trigger
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
builder.emitTrigger(newJob)
}
+ else {
+ // Set periodic trigger
+ Utilities.addPeriodicTrigger(newJob, '@daily')
+ }
}
}
}
[true, false].each { isPR ->
['Windows_NT'].each { os ->
['x64', 'x86'].each { arch ->
- ['ryujit', 'legacy_backend'].each { jit ->
+ ['ryujit'].each { jit ->
[true, false].each { pgo_optimized ->
- if (arch == 'x64' && jit == 'legacy_backend') {
- return
- }
-
- // pgo not supported for legacy_backend
- if (pgo_optimized && jit == 'legacy_backend') {
- return
- }
-
['full_opt', 'min_opt'].each { opt_level ->
def architecture = arch
// Save machinedata.json to /artifact/bin/ Jenkins dir
def archiveSettings = new ArchivalSettings()
archiveSettings.addFiles('throughput-*.csv')
+ archiveSettings.setAlwaysArchive()
Utilities.addArchival(newJob, archiveSettings)
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
builder.triggerForBranch(branch)
builder.emitTrigger(newJob)
}
- else {
+ else if (opt_level == 'full_opt' && pgo_optimized) {
// Set a push trigger
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
builder.emitTrigger(newJob)
}
+ else {
+ // Set periodic trigger
+ Utilities.addPeriodicTrigger(newJob, '@daily')
+ }
}
}
}
def newBuildJob = job(fullBuildJobName) {
steps {
shell("./build.sh verbose ${architecture} ${configuration}")
+ shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}")
}
}
Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
- Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
+ Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**")
}
// Actual perf testing on the following OSes
- def perfOSList = ['Ubuntu14.04']
+ def perfOSList = ['Ubuntu16.04']
perfOSList.each { os ->
def newJob = job(getFullPerfJobName(project, os, isPR)) {
- label('linux_clr_perf')
+ label('ubuntu_1604_clr_perf')
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
def osGroup = getOSGroup(os)
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
+ def uploadString = '-uploadToBenchview'
+
+ def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
steps {
- shell("./tests/scripts/perf-prep.sh")
+ shell("./tests/scripts/perf-prep.sh --nocorefx")
shell("./init-tools.sh")
copyArtifacts(fullBuildJobName) {
includePatterns("bin/**")
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
- shell("""./tests/scripts/run-xunit-perf.sh \\
- --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
- --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
- --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
- --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
- --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
- --runType=\"${runType}\" \\
- --benchViewOS=\"${os}\" \\
- --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
- --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
- --uploadToBenchview""")
- shell("mkdir -p bin/toArchive/sandbox/Logs/")
- shell("rsync -a bin/sandbox/Logs/Perf-*.* bin/toArchive/sandbox/Logs/")
+ shell("""python3 ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""")
}
}
def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('bin/toArchive/**')
+ archiveSettings.addFiles('bin/sandbox_logs/**')
archiveSettings.addFiles('machinedata.json')
+ archiveSettings.setAlwaysArchive()
Utilities.addArchival(newJob, archiveSettings)
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
throughputOptLevelList.each { opt_level ->
def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
- label('linux_clr_perf')
+ label('ubuntu_1604_clr_perf')
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
def archiveSettings = new ArchivalSettings()
archiveSettings.addFiles('throughput-*.csv')
archiveSettings.addFiles('machinedata.json')
+ archiveSettings.setAlwaysArchive()
Utilities.addArchival(newJob, archiveSettings)
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
steps {
// Batch
- batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
- def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
+ def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"
// Profile=Off
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\Off\\")
+ batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
// Profile=On
if (opt_level != 'min_opt') {
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\On\\")
+ batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
}
}
}
def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('bin/toArchive/**')
+ archiveSettings.addFiles('bin/sandbox_logs/**')
archiveSettings.addFiles('machinedata.json')
+ archiveSettings.setAlwaysArchive()
Utilities.addArchival(newJob, archiveSettings)
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
builder.triggerForBranch(branch)
builder.emitTrigger(newJob)
}
- else {
+ else if (opt_level == 'full_opt') {
// Set a push trigger
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
builder.emitTrigger(newJob)
}
+ else {
+ // Set periodic trigger
+ Utilities.addPeriodicTrigger(newJob, '@daily')
+ }
}
}
}
steps {
// Install nuget and get BenchView tools
- batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
def archiveSettings = new ArchivalSettings()
archiveSettings.addFiles('bin/toArchive/**')
archiveSettings.addFiles('machinedata.json')
+ archiveSettings.setAlwaysArchive()
Utilities.addArchival(newJob, archiveSettings)
Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
steps {
// Batch
- batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
- def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
+ def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
// Scenario: ILLink
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
+ batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
+ "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
}
}
def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('bin/toArchive/**')
+ archiveSettings.addFiles('bin/sandbox_logs/**')
archiveSettings.addFiles('machinedata.json')
+ archiveSettings.setAlwaysArchive()
// Set the label (currently we are only measuring size, therefore we are running on VM).
Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')