// Note that the parameters will be set as env variables so we cannot use names that conflict
// with the engineering system parameter names.
+//-------------------------- Globals ---------------------------------//
+
+def validTestFolders = [
+ 'Benchstones',
+ 'BenchmarksGame',
+ 'Bytemark',
+ 'Math',
+ 'Span',
+ 'first half',
+ 'last half'
+ ]
+
//--------------------- Windows Functions ----------------------------//
def windowsBuild(String arch, String config, String pgo, boolean isBaseline) {
}
bat "set __TestIntermediateDir=int&&.\\build.cmd -${config} -${arch} -skipbuildpackages ${pgoBuildFlag}"
+ bat "tests\\runtest.cmd ${config} ${arch} GenerateLayoutOnly"
+ bat "rd /s /q bin\\obj"
// Stash build artifacts. Stash tests in an additional stash to be used by Linux test runs
stash name: "nt-${arch}-${pgo}${baselineString}-build-artifacts", includes: 'bin/**'
stash name: "nt-${arch}-${pgo}${baselineString}-test-artifacts", includes: 'bin/tests/**'
}
-def windowsPerf(String arch, String config, String uploadString, String runType, String opt_level, String jit, String pgo, String scenario, boolean isBaseline, boolean isProfileOn) {
+def windowsPerf(String arch, String config, String uploadString, String runType, String opt_level, String jit, String pgo, String scenario, boolean isBaseline, boolean isProfileOn, String testFolder) {
withCredentials([string(credentialsId: 'CoreCLR Perf BenchView Sas', variable: 'BV_UPLOAD_SAS_TOKEN')]) {
checkout scm
String baselineString = ""
baselineString = "-baseline"
}
dir ('.') {
- unstash "nt-${arch}-${pgo}${baselineString}-build-artifacts"
+ unstash "nt-${arch}-${pgo}${baselineString}-test-artifacts"
unstash "benchview-tools"
unstash "metadata"
}
+ String test = ''
+ if (testFolder != 'all') {
+ test = testFolder
+ }
+
String pgoTestFlag = ((pgo == 'nopgo') ? '-nopgo' : '')
// We want to use the baseline metadata for baseline runs. We expect to find the submission metadata in
bat "py \".\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\""
bat ".\\init-tools.cmd"
- bat "run.cmd build -Project=\"tests\\build.proj\" -BuildOS=Windows_NT -BuildType=${config} -BuildArch=${arch} -BatchRestorePackages"
- bat "tests\\runtest.cmd ${config} ${arch} GenerateLayoutOnly"
// We run run-xunit-perf differently for each of the different job types
String runXUnitCommonArgs = "-arch ${arch} -configuration ${config} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} ${pgoTestFlag} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\""
if (scenario == 'perf') {
String runXUnitPerfCommonArgs = "${runXUnitCommonArgs} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
- String runXUnitPerflabArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\performance\\perflab\\Perflab -library"
+ if (test == 'first half' || testFolder == 'all')
+ {
+ String runXUnitPerflabArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\performance\\perflab\\Perflab -library"
- profileArg = isProfileOn ? "default+${profileArg}+gcapi" : profileArg
- bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerflabArgs} -collectionFlags ${profileArg}"
+ profileArg = isProfileOn ? "default+${profileArg}+gcapi" : profileArg
+ bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerflabArgs} -collectionFlags ${profileArg}"
+ }
- String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality"
- bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
+ if (test == 'first half') {
+
+ [
+ 'Burgers',
+ 'Devirtualization',
+ 'FractalPerf',
+ 'Inlining',
+ 'Layout'
+ ].each { benchmark ->
+ String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${benchmark}"
+ bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
+ }
+ }
+ else if (test == 'last half') {
+ [
+ 'Linq',
+ 'Roslyn',
+ 'SciMark',
+ 'Serialization',
+ 'V8'
+ ].each { benchmark ->
+ String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${benchmark}"
+ bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
+ }
+ }
+ else {
+ String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${test}"
+ bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
+ }
}
else if (scenario == 'jitbench') {
String runXUnitPerfCommonArgs = "${runXUnitCommonArgs} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
['x64', 'x86'].each { arch ->
['full_opt'].each { opt_level ->
[false].each { isBaseline ->
- String baseline = ""
- if (isBaseline) {
- baseline = " baseline"
- }
- if (isPR() || !isBaseline) {
- innerLoopTests["windows ${arch} ryujit ${opt_level} pgo${baseline} perf"] = {
- simpleNode('windows_server_2016_clr_perf', 180) {
- windowsPerf(arch, config, uploadString, runType, opt_level, 'ryujit', 'pgo', 'perf', isBaseline, true)
+ validTestFolders.each { benchmark ->
+ String baseline = ""
+ if (isBaseline) {
+ baseline = " baseline"
+ }
+ if (isPR() || !isBaseline) {
+ innerLoopTests["windows ${arch} ryujit ${opt_level} pgo ${benchmark}${baseline} perf"] = {
+ simpleNode('windows_server_2016_clr_perf', 180) {
+ windowsPerf(arch, config, uploadString, runType, opt_level, 'ryujit', 'pgo', 'perf', isBaseline, true, benchmark)
+ }
}
}
}
if (!isPR()) {
outerLoopTests["windows ${arch} ryujit full_opt pgo${baseline} jitbench"] = {
simpleNode('windows_server_2016_clr_perf', 180) {
- windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'jitbench', false)
+ windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'jitbench', false, false, '')
}
}
outerLoopTests["windows ${arch} ryujit full_opt pgo${baseline} illink"] = {
simpleNode('Windows_NT', '20170427-elevated') {
- windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'illink', false)
+ windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'illink', false, false, '')
}
}
[true, false].each { isProfileOn ->
outerLoopTests["windows ${arch} ${jit} ${opt_level} ${pgo_enabled} perf"] = {
simpleNode('windows_server_2016_clr_perf', 180) {
- windowsPerf(arch, config, uploadString, runType, opt_level, jit, pgo_enabled, 'perf', false, isProfileOn)
+ windowsPerf(arch, config, uploadString, runType, opt_level, jit, pgo_enabled, 'perf', false, isProfileOn, 'all')
}
}
projects / platform / upstream / coreclr.git / commitdiff