Add lean run support to CoreCLR perf (#11347)

In the xunit performance harness we added support for a "lean" run which
only runs the stopwatch to collect duration and none of the ETW
providers are turned on.  This adds support in the automation for doing
runs that use this lean feature.

diff --git a/perf.groovy b/perf.groovy
index 2d09c31c0aa287610629323b3dfa507e02810739..8dfcabc43d5ce12638435802394749d1bf1af4eb 100644 (file)
--- a/perf.groovy
+++ b/perf.groovy
@@ -62,8 +62,16 @@ def static getOSGroup(def os) {
                 {
                     parameters
                     {
-                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to one.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
-                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to one.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
+                    }
+                }
+                else
+                {
+                    parameters
+                    {
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
                     }
                 }
                 def configuration = 'Release'
@@ -96,8 +104,13 @@ def static getOSGroup(def os) {
 
                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 
+                        // Run with just stopwatch
                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}")
                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}")
+                        
+                        // Run with the full set of counters enabled
+                        batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags deafult+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
+                        batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags deafult+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
                     }
                 }
                 

diff --git a/tests/scripts/run-xunit-perf.cmd b/tests/scripts/run-xunit-perf.cmd
index 11279aeedaa2a672f07a586a93587ba47562844e..e279fa3f24ccfbea6f4e35fb3fbf83dd6392062f 100644 (file)
--- a/tests/scripts/run-xunit-perf.cmd
+++ b/tests/scripts/run-xunit-perf.cmd
@@ -18,11 +18,15 @@ setlocal
   set USAGE_DISPLAYED=
   set SHOULD_UPLOAD_TO_BENCHVIEW=
   set BENCHVIEW_PATH=
+  set COLLECTION_FLAGS=stopwatch
+  set ETW_COLLECTION=Off
+  set STABILITY_PREFIX=
 
   call :parse_command_line_arguments %*
   if defined USAGE_DISPLAYED exit /b %ERRORLEVEL%
 
   call :set_test_architecture  || exit /b 1
+  call :set_collection_config  || exit /b 1
   call :verify_benchview_tools || exit /b 1
   call :verify_core_overlay    || exit /b 1
   call :set_perf_run_log       || exit /b 1
@@ -86,7 +90,7 @@ setlocal
   if defined IS_SCENARIO_TEST (
     call :run_cmd corerun.exe "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1
   ) else (
-    call :run_cmd corerun.exe PerfHarness.dll "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1
+    call :run_cmd %STABILITY_PREFIX% corerun.exe PerfHarness.dll "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf --perf:collect %COLLECTION_FLAGS% 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1
   )
 
   IF %ERRORLEVEL% NEQ 0 (
@@ -118,6 +122,12 @@ rem ****************************************************************************
     shift
     goto :parse_command_line_arguments
   )
+  IF /I [%~1] == [-stabilityPrefix] (
+    set STABILITY_PREFIX=%~2
+    shift
+    shift
+    goto :parse_command_line_arguments
+  )
   IF /I [%~1] == [-scenarioTest] (
     set IS_SCENARIO_TEST=1
     shift
@@ -134,6 +144,12 @@ rem ****************************************************************************
     shift
     goto :parse_command_line_arguments
   )
+  IF /I [%~1] == [-collectionflags] (
+    set COLLECTION_FLAGS=%~2
+    shift
+    shift
+    goto :parse_command_line_arguments
+  )
   IF /I [%~1] == [-library] (
     set TEST_FILE_EXT=dll
     shift
@@ -208,6 +224,18 @@ rem ****************************************************************************
   )
   exit /b 0
 
+  :set_collection_config
+rem ****************************************************************************
+rem   Set's the config based on the providers used for collection
+rem ****************************************************************************
+  if /I [%COLLECTION_FLAGS%] == [stopwatch] (
+    set ETW_COLLECTION=Off
+  ) else (
+    set ETW_COLLECTION=On
+  )
+  exit /b 0
+
+  
 :set_perf_run_log
 rem ****************************************************************************
 rem   Sets the script's output log file.
@@ -266,6 +294,7 @@ setlocal
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config-name "%TEST_CONFIG%"
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config Configuration "%TEST_CONFIG%"
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config OS "Windows_NT"
+  set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config Profile "%ETW_COLLECTION%"
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --arch "%TEST_ARCHITECTURE%"
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --machinepool "PerfSnake"
   call :run_cmd py.exe "%BENCHVIEW_PATH%\submission.py" measurement.json %LV_SUBMISSION_ARGS%
@@ -288,7 +317,7 @@ rem ****************************************************************************
 rem   Script's usage.
 rem ****************************************************************************
   set USAGE_DISPLAYED=1
-  echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-generateBenchviewData] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^> [-scenarioTest]
+  echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-generateBenchviewData] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^> [-scenarioTest] [-collectionFlags] ^<default^+CacheMisses^+InstructionRetired^+BranchMispredictions^+gcapi^>
   echo/
   echo For the path to the tests you can pass a parent directory and the script will grovel for
   echo all tests in subdirectories and run them.
@@ -300,6 +329,10 @@ rem ****************************************************************************
   echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for
   echo PRs.
   echo -scenarioTest should be included if you are running a scenario benchmark.
+  echo -collectionFlags This is used to specify what collectoin flags get passed to the performance
+  echo harness that is doing the test running.  If this is not specified we only use stopwatch.
+  echo Other flags are "default", which is the whatever the test being run specified, "CacheMisses",
+  echo "BranchMispredictions", and "InstructionsRetired".  
   exit /b %ERRORLEVEL%
 
 :print_error

diff --git a/tests/src/Common/PerfHarness/project.json b/tests/src/Common/PerfHarness/project.json
index a1ecb2867e1a8bab6d10e6a109304aaa15e9f11b..bed9544b6539c4b7e9a00d38315ac3337b1d53b9 100644 (file)
--- a/tests/src/Common/PerfHarness/project.json
+++ b/tests/src/Common/PerfHarness/project.json
@@ -12,7 +12,7 @@
           "type": "platform",
           "version": "1.1.0"
         },
-        "xunit.performance.api": "1.0.0-beta-build0003"
+        "xunit.performance.api": "1.0.0-beta-build0004"
       }
     }
   }

diff --git a/tests/src/Common/external/project.json b/tests/src/Common/external/project.json
index 12c986a98be4e391837ce8e740201367e1a3f5bd..b5e378db94df6ec4bef2c1c13892be033ee6a367 100644 (file)
--- a/tests/src/Common/external/project.json
+++ b/tests/src/Common/external/project.json
@@ -1,10 +1,10 @@
 {
   "dependencies": {
     "Microsoft.CodeAnalysis.Compilers": "1.1.1",
-    "xunit.performance.api": "1.0.0-beta-build0003",
-    "xunit.performance.core": "1.0.0-beta-build0003",
-    "xunit.performance.execution": "1.0.0-beta-build0003",
-    "xunit.performance.metrics": "1.0.0-beta-build0003",
+    "xunit.performance.api": "1.0.0-beta-build0004",
+    "xunit.performance.core": "1.0.0-beta-build0004",
+    "xunit.performance.execution": "1.0.0-beta-build0004",
+    "xunit.performance.metrics": "1.0.0-beta-build0004",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Newtonsoft.Json": "9.0.1",
     "xunit": "2.2.0-beta2-build3300",

diff --git a/tests/src/JIT/config/benchmark+roslyn/project.json b/tests/src/JIT/config/benchmark+roslyn/project.json
index 6838ed1de881f63c35093f0a96b007b5f32ca5bd..d6e0d61c0e057a74aa32b0d27bc48d86cdc13c42 100644 (file)
--- a/tests/src/JIT/config/benchmark+roslyn/project.json
+++ b/tests/src/JIT/config/benchmark+roslyn/project.json
@@ -1,10 +1,10 @@
 {
   "dependencies": {
     "Microsoft.CodeAnalysis.Compilers": "1.1.1",
-    "xunit.performance.api": "1.0.0-beta-build0003",
-    "xunit.performance.core": "1.0.0-beta-build0003",
-    "xunit.performance.execution": "1.0.0-beta-build0003",
-    "xunit.performance.metrics": "1.0.0-beta-build0003",
+    "xunit.performance.api": "1.0.0-beta-build0004",
+    "xunit.performance.core": "1.0.0-beta-build0004",
+    "xunit.performance.execution": "1.0.0-beta-build0004",
+    "xunit.performance.metrics": "1.0.0-beta-build0004",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01",
     "System.Console": "4.4.0-beta-24913-02",

diff --git a/tests/src/JIT/config/benchmark+serialize/project.json b/tests/src/JIT/config/benchmark+serialize/project.json
index e450d6156f6f35e6657ab5a4c2061b7f1ce913ad..5c20a0970c26bb526d797016c8b6a936ba451f2d 100644 (file)
--- a/tests/src/JIT/config/benchmark+serialize/project.json
+++ b/tests/src/JIT/config/benchmark+serialize/project.json
@@ -1,9 +1,9 @@
 {
   "dependencies": {
-    "xunit.performance.api": "1.0.0-beta-build0003",
-    "xunit.performance.core": "1.0.0-beta-build0003",
-    "xunit.performance.execution": "1.0.0-beta-build0003",
-    "xunit.performance.metrics": "1.0.0-beta-build0003",
+    "xunit.performance.api": "1.0.0-beta-build0004",
+    "xunit.performance.core": "1.0.0-beta-build0004",
+    "xunit.performance.execution": "1.0.0-beta-build0004",
+    "xunit.performance.metrics": "1.0.0-beta-build0004",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01",
     "Newtonsoft.Json": "7.0.1",

diff --git a/tests/src/JIT/config/benchmark/project.json b/tests/src/JIT/config/benchmark/project.json
index 9c41a91713c80f52a099c0940ceee9c8eab0d18d..5f1fde013046d7e72424ac1143b2cab134f5bf28 100644 (file)
--- a/tests/src/JIT/config/benchmark/project.json
+++ b/tests/src/JIT/config/benchmark/project.json
@@ -1,9 +1,9 @@
 {
   "dependencies": {
-    "xunit.performance.api": "1.0.0-beta-build0003",
-    "xunit.performance.core": "1.0.0-beta-build0003",
-    "xunit.performance.execution": "1.0.0-beta-build0003",
-    "xunit.performance.metrics": "1.0.0-beta-build0003",
+    "xunit.performance.api": "1.0.0-beta-build0004",
+    "xunit.performance.core": "1.0.0-beta-build0004",
+    "xunit.performance.execution": "1.0.0-beta-build0004",
+    "xunit.performance.metrics": "1.0.0-beta-build0004",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01",
     "System.Collections.NonGeneric": "4.4.0-beta-24913-02",