[true, false].each { isPR ->
['Windows_NT'].each { os ->
['x64', 'x86'].each { arch ->
- [true, false].each { isSmoketest ->
- ['ryujit', 'legacy_backend'].each { jit ->
-
- if (arch == 'x64' && jit == 'legacy_backend') {
- return
- }
-
- ['full_opt', 'min_opt'].each { opt_level ->
- if (isSmoketest && opt_level == 'min_opt') {
- return
- }
-
- def architecture = arch
- def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
- def testEnv = ""
+ ['ryujit'].each { jit ->
+ ['full_opt', 'min_opt'].each { opt_level ->
- if (jit == 'legacy_backend') {
- testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
- }
+ def architecture = arch
+ def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
+ def testEnv = ""
- def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
- // Set the label.
- label('windows_server_2016_clr_perf')
- wrappers {
- credentialsBinding {
- string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
- }
+ def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
+ // Set the label.
+ label('windows_server_2016_clr_perf')
+ wrappers {
+ credentialsBinding {
+ string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
}
+ }
- if (isPR) {
- parameters {
- stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
- }
- }
- if (isSmoketest) {
- parameters {
- stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
- stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
- }
- }
- else {
- parameters {
- stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
- stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
- }
+ if (isPR) {
+ parameters {
+ stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
}
+ }
- def configuration = 'Release'
- def runType = isPR ? 'private' : 'rolling'
- def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
- def uploadString = isSmoketest ? '' : '-uploadToBenchview'
-
- steps {
- // Batch
+ parameters {
+ stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
+ stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
+ }
- batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
- batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
- batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
- //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
- //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
- batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
- "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
- "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
- "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
- "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
- batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
- batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
+ def configuration = 'Release'
+ def runType = isPR ? 'private' : 'rolling'
+ def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
+ def uploadString = '-uploadToBenchview'
- batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
+ steps {
+ // Batch
- def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
+ batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
+ batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+ //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+ //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+ batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+ "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
+ batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
+ batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
- // Run with just stopwatch: Profile=Off
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\Off\\")
+ batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\")
+ def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
- // Run with the full set of counters enabled: Profile=On
- if (opt_level != 'min_opt') {
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
+ // Run with just stopwatch: Profile=Off
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
- }
+ // Run with the full set of counters enabled: Profile=On
+ if (opt_level != 'min_opt') {
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
}
}
+ }
- if (isSmoketest) {
- Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
- }
-
- def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('bin/toArchive/**')
- archiveSettings.addFiles('machinedata.json')
+ def archiveSettings = new ArchivalSettings()
+ archiveSettings.addFiles('bin/sandbox_logs/**')
+ archiveSettings.addFiles('machinedata.json')
- Utilities.addArchival(newJob, archiveSettings)
- Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+ Utilities.addArchival(newJob, archiveSettings)
+ Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
- newJob.with {
- logRotator {
- artifactDaysToKeep(30)
- daysToKeep(30)
- artifactNumToKeep(200)
- numToKeep(200)
- }
- wrappers {
- timeout {
- absolute(240)
- }
- }
+ newJob.with {
+ logRotator {
+ artifactDaysToKeep(30)
+ daysToKeep(30)
+ artifactNumToKeep(200)
+ numToKeep(200)
}
-
- if (isPR) {
- TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
- if (isSmoketest) {
- builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
+ wrappers {
+ timeout {
+ absolute(240)
}
- else {
- builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
+ }
+ }
- def opts = ""
- if (opt_level == 'min_opt') {
- opts = '\\W+min_opts'
- }
- def jitt = ""
- if (jit != 'ryujit') {
- jitt = "\\W+${jit}"
- }
+ if (isPR) {
+ TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+ builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
- builder.triggerOnlyOnComment()
- builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
- }
- builder.triggerForBranch(branch)
- builder.emitTrigger(newJob)
+ def opts = ""
+ if (opt_level == 'min_opt') {
+ opts = '\\W+min_opts'
}
- else {
- // Set a push trigger
- TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
- builder.emitTrigger(newJob)
+ def jitt = ""
+ if (jit != 'ryujit') {
+ jitt = "\\W+${jit}"
}
+
+ builder.triggerOnlyOnComment()
+ builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
+ builder.triggerForBranch(branch)
+ builder.emitTrigger(newJob)
+ }
+ else if (opt_level == 'full_opt') {
+ // Set a push trigger
+ TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+ builder.emitTrigger(newJob)
+ }
+ else {
+ // Set periodic trigger
+ Utilities.addPeriodicTrigger(newJob, '@daily')
}
}
}
[true, false].each { isPR ->
['Windows_NT'].each { os ->
['x64', 'x86'].each { arch ->
- ['ryujit', 'legacy_backend'].each { jit ->
+ ['ryujit'].each { jit ->
[true, false].each { pgo_optimized ->
- if (arch == 'x64' && jit == 'legacy_backend') {
- return
- }
-
- // pgo not supported for legacy_backend
- if (pgo_optimized && jit == 'legacy_backend') {
- return
- }
-
['full_opt', 'min_opt'].each { opt_level ->
def architecture = arch
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
- "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
- "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
builder.triggerForBranch(branch)
builder.emitTrigger(newJob)
}
- else {
+ else if (opt_level == 'full_opt' && pgo_optimized) {
// Set a push trigger
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
builder.emitTrigger(newJob)
}
+ else {
+ // Set periodic trigger
+ Utilities.addPeriodicTrigger(newJob, '@daily')
+ }
}
}
}
// Actual perf testing on the following OSes
- def perfOSList = ['Ubuntu14.04']
+ def perfOSList = ['Ubuntu16.04']
perfOSList.each { os ->
def newJob = job(getFullPerfJobName(project, os, isPR)) {
- label('linux_clr_perf')
+ label('ubuntu_1604_clr_perf')
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
parameters {
// Cap the maximum number of iterations to 21.
- stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
- stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
+ stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
+ stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
}
}
}
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
- "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
+ "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
shell("""./tests/scripts/run-xunit-perf.sh \\
--testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
throughputOptLevelList.each { opt_level ->
def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
- label('linux_clr_perf')
+ label('ubuntu_1604_clr_perf')
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
}
}
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
- "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
+ "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
-arch \"${architecture}\" \\
[true, false].each { isPR ->
['Windows_NT'].each { os ->
['x64', 'x86'].each { arch ->
- ['ryujit', 'legacy_backend'].each { jit ->
-
- if (arch == 'x64' && jit == 'legacy_backend') {
- return
- }
-
- ['full_opt', 'min_opt'].each { opt_level ->
+ ['ryujit'].each { jit ->
+ ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
def architecture = arch
def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
def testEnv = ""
- if (jit == 'legacy_backend') {
- testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
- }
// Set the label.
label('windows_server_2016_clr_perf')
steps {
// Batch
- batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
- "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
- "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
- def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
+ def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
- // Scenario: JitBench
+ // Profile=Off
batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\")
+
+ // Profile=On
+ if (opt_level != 'min_opt') {
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
+ }
}
}
def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('bin/toArchive/**')
+ archiveSettings.addFiles('bin/sandbox_logs/**')
archiveSettings.addFiles('machinedata.json')
Utilities.addArchival(newJob, archiveSettings)
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests")
builder.triggerOnlyOnComment()
- builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}{$opts}${jitt}\\W+perf\\W+scenarios.*")
+ builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*")
builder.triggerForBranch(branch)
builder.emitTrigger(newJob)
}
- else {
+ else if (opt_level == 'full_opt') {
// Set a push trigger
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
builder.emitTrigger(newJob)
}
+ else {
+ // Set periodic trigger
+ Utilities.addPeriodicTrigger(newJob, '@daily')
+ }
+ }
+ }
+ }
+ }
+}
+
+// Setup size-on-disk test
+['Windows_NT'].each { os ->
+ ['x64', 'x86'].each { arch ->
+ def architecture = arch
+ def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) {
+
+ wrappers {
+ credentialsBinding {
+ string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+ }
+ }
+
+ def channel = 'master'
+ def configuration = 'Release'
+ def runType = 'rolling'
+ def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%'
+ def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
+ def coreRoot = "${testBin}\\Tests\\Core_Root"
+ def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
+
+ steps {
+ // Install nuget and get BenchView tools
+ batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
+ batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+
+ // Generate submission metadata for BenchView
+ // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+ // we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+ batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+ "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
+ "py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
+ "py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
+
+ // Generate machine data from BenchView
+ batchFile("py \"${benchViewTools}\\machinedata.py\"")
+
+ // Build CoreCLR and gnerate test layout
+ batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
+ batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
+
+ // Run the size on disk benchmark
+ batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")
+
+ // From sodbench.csv, create measurment.json, then submission.json
+ batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
+ batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
+
+ // If this is a PR, upload submission.json
+ batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
+ }
+ }
+
+ Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
+
+ def archiveSettings = new ArchivalSettings()
+ archiveSettings.addFiles('bin/toArchive/**')
+ archiveSettings.addFiles('machinedata.json')
+
+ Utilities.addArchival(newJob, archiveSettings)
+ Utilities.standardJobSetup(newJob, project, false, "*/${branch}")
+
+ // Set the cron job here. We run nightly on each flavor, regardless of code changes
+ Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/)
+
+ newJob.with {
+ logRotator {
+ artifactDaysToKeep(30)
+ daysToKeep(30)
+ artifactNumToKeep(200)
+ numToKeep(200)
+ }
+ wrappers {
+ timeout {
+ absolute(240)
}
}
}
steps {
// Batch
- batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
- "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
- "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
- def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
+ def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest"
// Scenario: ILLink
batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
}
}
def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('bin/toArchive/**')
+ archiveSettings.addFiles('bin/sandbox_logs/**')
archiveSettings.addFiles('machinedata.json')
// Set the label (currently we are only measuring size, therefore we are running on VM).
projects / platform / upstream / coreclr.git / blobdiff