Add lean run support to CoreCLR perf (#11347)

In the xunit performance harness we added support for a "lean" run which
only runs the stopwatch to collect duration and none of the ETW
providers are turned on.  This adds support in the automation for doing
runs that use this lean feature.

diff --git a/perf.groovy b/perf.groovy
index 2d09c31..8dfcabc 100644 (file)
--- a/perf.groovy
+++ b/perf.groovy
@@ -62,8 +62,16 @@ def static getOSGroup(def os) {
                 {
                     parameters
                     {
-                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to one.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
-                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to one.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
+                    }
+                }
+                else
+                {
+                    parameters
+                    {
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
                     }
                 }
                 def configuration = 'Release'
@@ -96,8 +104,13 @@ def static getOSGroup(def os) {
 
                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 
+                        // Run with just stopwatch
                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}")
                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}")
+                        
+                        // Run with the full set of counters enabled
+                        batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags deafult+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
+                        batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags deafult+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
                     }
                 }
                 

diff --git a/tests/scripts/run-xunit-perf.cmd b/tests/scripts/run-xunit-perf.cmd
index 11279ae..e279fa3 100644 (file)
--- a/tests/scripts/run-xunit-perf.cmd
+++ b/tests/scripts/run-xunit-perf.cmd
@@ -18,11 +18,15 @@ setlocal
   set USAGE_DISPLAYED=
   set SHOULD_UPLOAD_TO_BENCHVIEW=
   set BENCHVIEW_PATH=
+  set COLLECTION_FLAGS=stopwatch
+  set ETW_COLLECTION=Off
+  set STABILITY_PREFIX=
 
   call :parse_command_line_arguments %*
   if defined USAGE_DISPLAYED exit /b %ERRORLEVEL%
 
   call :set_test_architecture  || exit /b 1
+  call :set_collection_config  || exit /b 1
   call :verify_benchview_tools || exit /b 1
   call :verify_core_overlay    || exit /b 1
   call :set_perf_run_log       || exit /b 1
@@ -86,7 +90,7 @@ setlocal
   if defined IS_SCENARIO_TEST (
     call :run_cmd corerun.exe "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1
   ) else (
-    call :run_cmd corerun.exe PerfHarness.dll "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1
+    call :run_cmd %STABILITY_PREFIX% corerun.exe PerfHarness.dll "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf --perf:collect %COLLECTION_FLAGS% 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1
   )
 
   IF %ERRORLEVEL% NEQ 0 (
@@ -118,6 +122,12 @@ rem ****************************************************************************
     shift
     goto :parse_command_line_arguments
   )
+  IF /I [%~1] == [-stabilityPrefix] (
+    set STABILITY_PREFIX=%~2
+    shift
+    shift
+    goto :parse_command_line_arguments
+  )
   IF /I [%~1] == [-scenarioTest] (
     set IS_SCENARIO_TEST=1
     shift
@@ -134,6 +144,12 @@ rem ****************************************************************************
     shift
     goto :parse_command_line_arguments
   )
+  IF /I [%~1] == [-collectionflags] (
+    set COLLECTION_FLAGS=%~2
+    shift
+    shift
+    goto :parse_command_line_arguments
+  )
   IF /I [%~1] == [-library] (
     set TEST_FILE_EXT=dll
     shift
@@ -208,6 +224,18 @@ rem ****************************************************************************
   )
   exit /b 0
 
+  :set_collection_config
+rem ****************************************************************************
+rem   Set's the config based on the providers used for collection
+rem ****************************************************************************
+  if /I [%COLLECTION_FLAGS%] == [stopwatch] (
+    set ETW_COLLECTION=Off
+  ) else (
+    set ETW_COLLECTION=On
+  )
+  exit /b 0
+
+  
 :set_perf_run_log
 rem ****************************************************************************
 rem   Sets the script's output log file.
@@ -266,6 +294,7 @@ setlocal
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config-name "%TEST_CONFIG%"
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config Configuration "%TEST_CONFIG%"
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config OS "Windows_NT"
+  set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --config Profile "%ETW_COLLECTION%"
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --arch "%TEST_ARCHITECTURE%"
   set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --machinepool "PerfSnake"
   call :run_cmd py.exe "%BENCHVIEW_PATH%\submission.py" measurement.json %LV_SUBMISSION_ARGS%
@@ -288,7 +317,7 @@ rem ****************************************************************************
 rem   Script's usage.
 rem ****************************************************************************
   set USAGE_DISPLAYED=1
-  echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-generateBenchviewData] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^> [-scenarioTest]
+  echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-generateBenchviewData] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^> [-scenarioTest] [-collectionFlags] ^<default^+CacheMisses^+InstructionRetired^+BranchMispredictions^+gcapi^>
   echo/
   echo For the path to the tests you can pass a parent directory and the script will grovel for
   echo all tests in subdirectories and run them.
@@ -300,6 +329,10 @@ rem ****************************************************************************
   echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for
   echo PRs.
   echo -scenarioTest should be included if you are running a scenario benchmark.
+  echo -collectionFlags This is used to specify what collectoin flags get passed to the performance
+  echo harness that is doing the test running.  If this is not specified we only use stopwatch.
+  echo Other flags are "default", which is the whatever the test being run specified, "CacheMisses",
+  echo "BranchMispredictions", and "InstructionsRetired".  
   exit /b %ERRORLEVEL%
 
 :print_error

diff --git a/tests/src/Common/PerfHarness/project.json b/tests/src/Common/PerfHarness/project.json
index a1ecb28..bed9544 100644 (file)
--- a/tests/src/Common/PerfHarness/project.json
+++ b/tests/src/Common/PerfHarness/project.json
@@ -12,7 +12,7 @@
           "type": "platform",
           "version": "1.1.0"
         },
-        "xunit.performance.api": "1.0.0-beta-build0003"
+        "xunit.performance.api": "1.0.0-beta-build0004"
       }
     }
   }

diff --git a/tests/src/Common/external/project.json b/tests/src/Common/external/project.json
index 12c986a..b5e378d 100644 (file)
--- a/tests/src/Common/external/project.json
+++ b/tests/src/Common/external/project.json
@@ -1,10 +1,10 @@
 {
   "dependencies": {
     "Microsoft.CodeAnalysis.Compilers": "1.1.1",
-    "xunit.performance.api": "1.0.0-beta-build0003",
-    "xunit.performance.core": "1.0.0-beta-build0003",
-    "xunit.performance.execution": "1.0.0-beta-build0003",
-    "xunit.performance.metrics": "1.0.0-beta-build0003",
+    "xunit.performance.api": "1.0.0-beta-build0004",
+    "xunit.performance.core": "1.0.0-beta-build0004",
+    "xunit.performance.execution": "1.0.0-beta-build0004",
+    "xunit.performance.metrics": "1.0.0-beta-build0004",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Newtonsoft.Json": "9.0.1",
     "xunit": "2.2.0-beta2-build3300",

diff --git a/tests/src/JIT/config/benchmark+roslyn/project.json b/tests/src/JIT/config/benchmark+roslyn/project.json
index 6838ed1..d6e0d61 100644 (file)
--- a/tests/src/JIT/config/benchmark+roslyn/project.json
+++ b/tests/src/JIT/config/benchmark+roslyn/project.json
@@ -1,10 +1,10 @@
 {
   "dependencies": {
     "Microsoft.CodeAnalysis.Compilers": "1.1.1",
-    "xunit.performance.api": "1.0.0-beta-build0003",
-    "xunit.performance.core": "1.0.0-beta-build0003",
-    "xunit.performance.execution": "1.0.0-beta-build0003",
-    "xunit.performance.metrics": "1.0.0-beta-build0003",
+    "xunit.performance.api": "1.0.0-beta-build0004",
+    "xunit.performance.core": "1.0.0-beta-build0004",
+    "xunit.performance.execution": "1.0.0-beta-build0004",
+    "xunit.performance.metrics": "1.0.0-beta-build0004",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01",
     "System.Console": "4.4.0-beta-24913-02",

diff --git a/tests/src/JIT/config/benchmark+serialize/project.json b/tests/src/JIT/config/benchmark+serialize/project.json
index e450d61..5c20a09 100644 (file)
--- a/tests/src/JIT/config/benchmark+serialize/project.json
+++ b/tests/src/JIT/config/benchmark+serialize/project.json
@@ -1,9 +1,9 @@
 {
   "dependencies": {
-    "xunit.performance.api": "1.0.0-beta-build0003",
-    "xunit.performance.core": "1.0.0-beta-build0003",
-    "xunit.performance.execution": "1.0.0-beta-build0003",
-    "xunit.performance.metrics": "1.0.0-beta-build0003",
+    "xunit.performance.api": "1.0.0-beta-build0004",
+    "xunit.performance.core": "1.0.0-beta-build0004",
+    "xunit.performance.execution": "1.0.0-beta-build0004",
+    "xunit.performance.metrics": "1.0.0-beta-build0004",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01",
     "Newtonsoft.Json": "7.0.1",

diff --git a/tests/src/JIT/config/benchmark/project.json b/tests/src/JIT/config/benchmark/project.json
index 9c41a91..5f1fde0 100644 (file)
--- a/tests/src/JIT/config/benchmark/project.json
+++ b/tests/src/JIT/config/benchmark/project.json
@@ -1,9 +1,9 @@
 {
   "dependencies": {
-    "xunit.performance.api": "1.0.0-beta-build0003",
-    "xunit.performance.core": "1.0.0-beta-build0003",
-    "xunit.performance.execution": "1.0.0-beta-build0003",
-    "xunit.performance.metrics": "1.0.0-beta-build0003",
+    "xunit.performance.api": "1.0.0-beta-build0004",
+    "xunit.performance.core": "1.0.0-beta-build0004",
+    "xunit.performance.execution": "1.0.0-beta-build0004",
+    "xunit.performance.metrics": "1.0.0-beta-build0004",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview2-25305-01",
     "System.Collections.NonGeneric": "4.4.0-beta-24913-02",