From: Drew Scoggins Date: Fri, 5 May 2017 17:39:15 +0000 (+0900) Subject: Add lean run support to CoreCLR perf (#11347) X-Git-Tag: accepted/tizen/base/20180629.140029~1164 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=803b48ca0f6e3097a3453d1d15d3765704461651;p=platform%2Fupstream%2Fcoreclr.git Add lean run support to CoreCLR perf (#11347) In the xunit performance harness we added support for a "lean" run which only runs the stopwatch to collect duration and none of the ETW providers are turned on. This adds support in the automation for doing runs that use this lean feature. --- diff --git a/perf.groovy b/perf.groovy index 2d09c31..8dfcabc 100644 --- a/perf.groovy +++ b/perf.groovy @@ -62,8 +62,16 @@ def static getOSGroup(def os) { { parameters { - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to one. We want to do this so that we can run as fast as possible as this is just for smoke testing') - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to one. We want to do this so that we can run as fast as possible as this is just for smoke testing') + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing') + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing') + } + } + else + { + parameters + { + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample') + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample') } } def configuration = 'Release' @@ -96,8 +104,13 @@ def static getOSGroup(def os) { batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") + // Run with just stopwatch batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}") batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}") + + // Run with the full set of counters enabled + batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags deafult+BranchMispredictions+CacheMisses+InstructionRetired+gcapi") + batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags deafult+BranchMispredictions+CacheMisses+InstructionRetired+gcapi") } } diff --git a/tests/scripts/run-xunit-perf.cmd b/tests/scripts/run-xunit-perf.cmd index 11279ae..e279fa3 100644 --- a/tests/scripts/run-xunit-perf.cmd +++ b/tests/scripts/run-xunit-perf.cmd @@ -18,11 +18,15 @@ setlocal set USAGE_DISPLAYED= set SHOULD_UPLOAD_TO_BENCHVIEW= set BENCHVIEW_PATH= + set COLLECTION_FLAGS=stopwatch + set ETW_COLLECTION=Off + set STABILITY_PREFIX= call :parse_command_line_arguments %* if defined USAGE_DISPLAYED exit /b %ERRORLEVEL% call :set_test_architecture || exit /b 1 + call :set_collection_config || exit /b 1 call :verify_benchview_tools || exit /b 1 call :verify_core_overlay || exit /b 1 call :set_perf_run_log || exit /b 1 @@ -86,7 +90,7 @@ setlocal if defined IS_SCENARIO_TEST ( call :run_cmd corerun.exe "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1 ) else ( - call :run_cmd corerun.exe PerfHarness.dll "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1 + call :run_cmd %STABILITY_PREFIX% corerun.exe PerfHarness.dll "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf --perf:collect %COLLECTION_FLAGS% 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1 ) IF %ERRORLEVEL% NEQ 0 ( @@ -118,6 +122,12 @@ rem **************************************************************************** shift goto :parse_command_line_arguments ) + IF /I [%~1] == [-stabilityPrefix] ( + set STABILITY_PREFIX=%~2 + shift + shift + goto :parse_command_line_arguments + ) IF /I [%~1] == [-scenarioTest] ( set IS_SCENARIO_TEST=1 shift @@ -134,6 +144,12 @@ rem **************************************************************************** shift goto :parse_command_line_arguments ) + IF /I [%~1] == [-collectionflags] ( + set COLLECTION_FLAGS=%~2 + shift + shift + goto :parse_command_line_arguments + ) IF /I [%~1] == [-library] ( set TEST_FILE_EXT=dll shift @@ -208,6 +224,18 @@ rem **************************************************************************** ) exit /b 0 + :set_collection_config +rem **************************************************************************** +rem Set's the config based on the providers used for collection +rem **************************************************************************** + if /I [%COLLECTION_FLAGS%] == [stopwatch] ( + set ETW_COLLECTION=Off + ) else ( + set ETW_COLLECTION=On + ) + exit /b 0 + + :set_perf_run_log rem **************************************************************************** rem Sets the script's output log file. @@ -266,6 +294,7 @@ setlocal set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config-name "%TEST_CONFIG%" set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config Configuration "%TEST_CONFIG%" set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config OS "Windows_NT" + set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config Profile "%ETW_COLLECTION%" set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --arch "%TEST_ARCHITECTURE%" set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --machinepool "PerfSnake" call :run_cmd py.exe "%BENCHVIEW_PATH%\submission.py" measurement.json %LV_SUBMISSION_ARGS% @@ -288,7 +317,7 @@ rem **************************************************************************** rem Script's usage. rem **************************************************************************** set USAGE_DISPLAYED=1 - echo run-xunit-perf.cmd -testBinLoc ^ [-library] [-arch] ^ [-configuration] ^ [-generateBenchviewData] ^ [-runtype] ^ [-scenarioTest] + echo run-xunit-perf.cmd -testBinLoc ^ [-library] [-arch] ^ [-configuration] ^ [-generateBenchviewData] ^ [-runtype] ^ [-scenarioTest] [-collectionFlags] ^ echo/ echo For the path to the tests you can pass a parent directory and the script will grovel for echo all tests in subdirectories and run them. @@ -300,6 +329,10 @@ rem **************************************************************************** echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for echo PRs. echo -scenarioTest should be included if you are running a scenario benchmark. + echo -collectionFlags This is used to specify what collectoin flags get passed to the performance + echo harness that is doing the test running. If this is not specified we only use stopwatch. + echo Other flags are "default", which is the whatever the test being run specified, "CacheMisses", + echo "BranchMispredictions", and "InstructionsRetired". exit /b %ERRORLEVEL% :print_error diff --git a/tests/src/Common/PerfHarness/project.json b/tests/src/Common/PerfHarness/project.json index a1ecb28..bed9544 100644 --- a/tests/src/Common/PerfHarness/project.json +++ b/tests/src/Common/PerfHarness/project.json @@ -12,7 +12,7 @@ "type": "platform", "version": "1.1.0" }, - "xunit.performance.api": "1.0.0-beta-build0003" + "xunit.performance.api": "1.0.0-beta-build0004" } } } diff --git a/tests/src/Common/external/project.json b/tests/src/Common/external/project.json index 12c986a..b5e378d 100644 --- a/tests/src/Common/external/project.json +++ b/tests/src/Common/external/project.json @@ -1,10 +1,10 @@ { "dependencies": { "Microsoft.CodeAnalysis.Compilers": "1.1.1", - "xunit.performance.api": "1.0.0-beta-build0003", - "xunit.performance.core": "1.0.0-beta-build0003", - "xunit.performance.execution": "1.0.0-beta-build0003", - "xunit.performance.metrics": "1.0.0-beta-build0003", + "xunit.performance.api": "1.0.0-beta-build0004", + "xunit.performance.core": "1.0.0-beta-build0004", + "xunit.performance.execution": "1.0.0-beta-build0004", + "xunit.performance.metrics": "1.0.0-beta-build0004", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Newtonsoft.Json": "9.0.1", "xunit": "2.2.0-beta2-build3300", diff --git a/tests/src/JIT/config/benchmark+roslyn/project.json b/tests/src/JIT/config/benchmark+roslyn/project.json index 6838ed1..d6e0d61 100644 --- a/tests/src/JIT/config/benchmark+roslyn/project.json +++ b/tests/src/JIT/config/benchmark+roslyn/project.json @@ -1,10 +1,10 @@ { "dependencies": { "Microsoft.CodeAnalysis.Compilers": "1.1.1", - "xunit.performance.api": "1.0.0-beta-build0003", - "xunit.performance.core": "1.0.0-beta-build0003", - "xunit.performance.execution": "1.0.0-beta-build0003", - "xunit.performance.metrics": "1.0.0-beta-build0003", + "xunit.performance.api": "1.0.0-beta-build0004", + "xunit.performance.core": "1.0.0-beta-build0004", + "xunit.performance.execution": "1.0.0-beta-build0004", + "xunit.performance.metrics": "1.0.0-beta-build0004", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01", "System.Console": "4.4.0-beta-24913-02", diff --git a/tests/src/JIT/config/benchmark+serialize/project.json b/tests/src/JIT/config/benchmark+serialize/project.json index e450d61..5c20a09 100644 --- a/tests/src/JIT/config/benchmark+serialize/project.json +++ b/tests/src/JIT/config/benchmark+serialize/project.json @@ -1,9 +1,9 @@ { "dependencies": { - "xunit.performance.api": "1.0.0-beta-build0003", - "xunit.performance.core": "1.0.0-beta-build0003", - "xunit.performance.execution": "1.0.0-beta-build0003", - "xunit.performance.metrics": "1.0.0-beta-build0003", + "xunit.performance.api": "1.0.0-beta-build0004", + "xunit.performance.core": "1.0.0-beta-build0004", + "xunit.performance.execution": "1.0.0-beta-build0004", + "xunit.performance.metrics": "1.0.0-beta-build0004", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01", "Newtonsoft.Json": "7.0.1", diff --git a/tests/src/JIT/config/benchmark/project.json b/tests/src/JIT/config/benchmark/project.json index 9c41a91..5f1fde0 100644 --- a/tests/src/JIT/config/benchmark/project.json +++ b/tests/src/JIT/config/benchmark/project.json @@ -1,9 +1,9 @@ { "dependencies": { - "xunit.performance.api": "1.0.0-beta-build0003", - "xunit.performance.core": "1.0.0-beta-build0003", - "xunit.performance.execution": "1.0.0-beta-build0003", - "xunit.performance.metrics": "1.0.0-beta-build0003", + "xunit.performance.api": "1.0.0-beta-build0004", + "xunit.performance.core": "1.0.0-beta-build0004", + "xunit.performance.execution": "1.0.0-beta-build0004", + "xunit.performance.metrics": "1.0.0-beta-build0004", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01", "System.Collections.NonGeneric": "4.4.0-beta-24913-02",