[true, false].each { isPR ->
['Windows_NT'].each { os ->
['x64', 'x86'].each { arch ->
- def architecture = arch
+ ['full_opt', 'min_opt'].each { opt_level ->
+ def architecture = arch
- def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
- // Set the label.
- label('windows_clr_perf')
- wrappers {
- credentialsBinding {
- string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+ def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}", isPR)) {
+ // Set the label.
+ label('windows_clr_perf')
+ wrappers {
+ credentialsBinding {
+ string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+ }
}
- }
- if (isPR)
- {
- parameters
- {
- stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
- }
- }
- def configuration = 'Release'
- def runType = isPR ? 'private' : 'rolling'
- def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
+ if (isPR)
+ {
+ parameters
+ {
+ stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
+ }
+ }
+ def configuration = 'Release'
+ def runType = isPR ? 'private' : 'rolling'
+ def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
- steps {
- // Batch
+ steps {
+ // Batch
- batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
- batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
- batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
- batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
- //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
- //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
- batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
- "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
- "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
- "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
- "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
- batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
- batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
- batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
- batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
+ batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
+ batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
+ batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+ batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+ //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+ //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+ batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+ "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
+ batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
+ batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
+ batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
+ batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
+ }
}
- }
- // Save machinedata.json to /artifact/bin/ Jenkins dir
- def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('throughput-*.csv')
- Utilities.addArchival(newJob, archiveSettings)
+ // Save machinedata.json to /artifact/bin/ Jenkins dir
+ def archiveSettings = new ArchivalSettings()
+ archiveSettings.addFiles('throughput-*.csv')
+ Utilities.addArchival(newJob, archiveSettings)
- Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+ Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
- if (isPR) {
- TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
- builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
- builder.triggerOnlyOnComment()
- builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
- builder.triggerForBranch(branch)
- builder.emitTrigger(newJob)
- }
- else {
- // Set a push trigger
- TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
- builder.emitTrigger(newJob)
+ if (isPR) {
+ def opts = ""
+ if (opt_level == 'min_opts')
+ {
+ opts = '\\W+min_opts'
+ }
+ TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+ builder.setGithubContext("${os} ${arch} ${opt_level} CoreCLR Throughput Perf Tests")
+ builder.triggerOnlyOnComment()
+ builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}\\W+throughput.*")
+ builder.triggerForBranch(branch)
+ builder.emitTrigger(newJob)
+ }
+ else {
+ // Set a push trigger
+ TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+ builder.emitTrigger(newJob)
+ }
}
}
}
// Actual perf testing on the following OSes
def throughputOSList = ['Ubuntu14.04']
+ def throughputOptLevelList = ['full_opt', 'min_opt']
+
+ def throughputOSOptLevelList = []
+
throughputOSList.each { os ->
- def newJob = job(getFullThroughputJobName(project, os, isPR)) {
+ throughputOptLevelList.each { opt_level ->
+ throughputOSOptLevelList.add("${os}_${opt_level}")
+ }
+ }
- label('linux_clr_perf')
- wrappers {
- credentialsBinding {
- string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+ throughputOSList.each { os ->
+ throughputOptLevelList.each { opt_level ->
+ def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
+
+ label('linux_clr_perf')
+ wrappers {
+ credentialsBinding {
+ string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+ }
}
- }
- if (isPR)
- {
- parameters
+ if (isPR)
{
- stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
+ parameters
+ {
+ stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
+ }
}
- }
- parameters {
- stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
- }
+ parameters {
+ stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
+ }
- def osGroup = getOSGroup(os)
- def runType = isPR ? 'private' : 'rolling'
- def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
+ def osGroup = getOSGroup(os)
+ def runType = isPR ? 'private' : 'rolling'
+ def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
- steps {
- shell("bash ./tests/scripts/perf-prep.sh --throughput")
- shell("./init-tools.sh")
- copyArtifacts(fullBuildJobName) {
- includePatterns("bin/Product/**")
- buildSelector {
- buildNumber('\${PRODUCT_BUILD}')
+ steps {
+ shell("bash ./tests/scripts/perf-prep.sh --throughput")
+ shell("./init-tools.sh")
+ copyArtifacts(fullBuildJobName) {
+ includePatterns("bin/Product/**")
+ buildSelector {
+ buildNumber('\${PRODUCT_BUILD}')
+ }
}
+ shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
+ "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
+ "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
+ shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
+ -arch \"${architecture}\" \\
+ -os \"${os}\" \\
+ -configuration \"${configuration}\" \\
+ -opt_level \"${opt_level}\" \\
+ -clr_root \"\${WORKSPACE}\" \\
+ -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
+ -run_type \"${runType}\" \\
+ -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
}
- shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
- "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
- "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
- shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
- -arch \"${architecture}\" \\
- -os \"${os}\" \\
- -configuration \"${configuration}\" \\
- -clr_root \"\${WORKSPACE}\" \\
- -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
- -run_type \"${runType}\" \\
- -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
}
- }
- // Save machinedata.json to /artifact/bin/ Jenkins dir
- def archiveSettings = new ArchivalSettings()
- archiveSettings.addFiles('throughput-*.csv')
- archiveSettings.addFiles('machinedata.json')
- Utilities.addArchival(newJob, archiveSettings)
+ // Save machinedata.json to /artifact/bin/ Jenkins dir
+ def archiveSettings = new ArchivalSettings()
+ archiveSettings.addFiles('throughput-*.csv')
+ archiveSettings.addFiles('machinedata.json')
+ Utilities.addArchival(newJob, archiveSettings)
- Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+ Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
- // For perf, we need to keep the run results longer
- newJob.with {
- // Enable the log rotator
- logRotator {
- artifactDaysToKeep(7)
- daysToKeep(300)
- artifactNumToKeep(25)
- numToKeep(1000)
+ // For perf, we need to keep the run results longer
+ newJob.with {
+ // Enable the log rotator
+ logRotator {
+ artifactDaysToKeep(7)
+ daysToKeep(300)
+ artifactNumToKeep(25)
+ numToKeep(1000)
+ }
}
- }
+ } // opt_level
} // os
- def flowJobTPRunList = throughputOSList.collect { os ->
+ def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
"{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
}
def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
parser.add_argument('-assembly_root', dest='assembly_root', default=None)
parser.add_argument('-benchview_path', dest='benchview_path', default=None)
parser.add_argument('-iterations', dest='iterations', default=5, type=int)
+parser.add_argument('-opt_level', dest='opt_level', default='full_opt')
##########################################################################
# Helper Functions
assembly_root = args.assembly_root
benchview_path = args.benchview_path
iterations = args.iterations
+ opt_level = args.opt_level.lower()
def validate_arg(arg, check):
""" Validate an individual arg
valid_build_types = ['Release']
valid_run_types = ['rolling', 'private']
valid_os = ['Windows_NT', 'Ubuntu14.04']
+ valid_opt_levels = ['full_opt', 'min_opt']
arch = next((a for a in valid_archs if a.lower() == arch.lower()), arch)
build_type = next((b for b in valid_build_types if b.lower() == build_type.lower()), build_type)
validate_arg(build_type, lambda item: item in valid_build_types)
validate_arg(run_type, lambda item: item in valid_run_types)
validate_arg(iterations, lambda item: item > 0)
+ validate_arg(opt_level, lambda item: item in valid_opt_levels)
if clr_root is None:
raise Exception('--clr_root must be set')
benchview_path = os.path.normpath(benchview_path)
validate_arg(benchview_path, lambda item: os.path.isdir(benchview_path))
- args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations)
+ args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations, opt_level)
# Log configuration
log('Configuration:')
log(' os: %s' % operating_system)
log(' os_group: %s' % os_group)
log(' build_type: %s' % build_type)
+ log(' opt_level: %s' % opt_level)
log(' run_type: %s' % run_type)
log(' iterations: %d' % iterations)
log(' clr_root: %s' % clr_root)
return csv_file_name
-def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assemblies_path):
+def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assemblies_path, opt_level):
""" Run throughput testing for a given dll
Args:
dll_name: the name of the dll
dll_path
]
+ my_env = os.environ
+
+ if opt_level == 'min_opt':
+ my_env['COMPlus_JITMinOpts'] = '1'
+
log(" ".join(run_args))
# Time.clock() returns seconds, with a resolution of 0.4 microseconds, so multiply by the multiplier to get milliseconds
multiplier = 1000
for iteration in range(0,iterations + 1):
- proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env)
start_time = timeit.default_timer()
(out, err) = proc.communicate()
global os_group_list
global python_exe_list
- architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations = validate_args(args)
+ architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations, opt_level = validate_args(args)
arch = architecture
current_dir = os.getcwd()
(not dll_file_name in dll_exclude_list["All"])):
dll_name = dll_file_name.replace(".dll", "")
dll_path = os.path.join(assembly_root, dll_file_name)
- dll_elapsed_times = runIterations(dll_file_name, dll_path, iterations, crossgen_path, jit_path, assembly_root)
+ dll_elapsed_times = runIterations(dll_file_name, dll_path, iterations, crossgen_path, jit_path, assembly_root, opt_level)
if len(dll_elapsed_times) != 0:
if not benchview_path is None:
"--config",
"OS",
operating_system,
+ "--config",
+ "OptLevel",
+ opt_level,
"--arch",
architecture,
"--machinepool",
projects / platform / upstream / coreclr.git / commitdiff