Add scenario-based tests

Modified Xunit runner to run Scenario Benchmarks
Updating xunit-performance packages

diff --git a/tests/scripts/run-xunit-perf.cmd b/tests/scripts/run-xunit-perf.cmd
index 0d657c6e9e11cb42cfc0a60e02e3db64912f7fe1..36456322db59ac9e6a6f4e225315d0746c3e5227 100644 (file)
--- a/tests/scripts/run-xunit-perf.cmd
+++ b/tests/scripts/run-xunit-perf.cmd
@@ -12,6 +12,7 @@ set TEST_FILE_EXT=exe
 set TEST_ARCH=x64
 set TEST_ARCHITECTURE=x64
 set TEST_CONFIG=Release
+set IS_SCENARIO_TEST=0
 
 goto :ARGLOOP
 
@@ -34,7 +35,6 @@ if NOT EXIST %CORECLR_OVERLAY% (
 )
 
 @echo --- setting up sandbox
-
 if exist sandbox rd /s /q sandbox
 if exist sandbox echo ERROR: Failed to remove the sandbox folder& exit /b 1
 if not exist sandbox mkdir sandbox
@@ -94,11 +94,19 @@ if DEFINED TEST_ENV (
     )
 )
 
-corerun.exe PerfHarness.dll %CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT% --perf:runid Perf > %BENCHNAME%.out
+if [%IS_SCENARIO_TEST%] == [1] (
+  corerun.exe %CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT% --perf:runid Perf > %BENCHNAME%.out
+) else (
+  corerun.exe PerfHarness.dll %CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT% --perf:runid Perf > %BENCHNAME%.out
+)
 
 @rem optionally generate results for benchview
 if not [%BENCHVIEW_PATH%] == [] (
-  py "%BENCHVIEW_PATH%\measurement.py" xunit "Perf-%BENCHNAME%.xml" --better desc --drop-first-value --append
+  if [%IS_SCENARIO_TEST%] == [1] (
+    py "%BENCHVIEW_PATH%\measurement.py" xunitscenario "Perf-%BENCHNAME%.xml" --better desc --drop-first-value --append
+  ) else (
+    py "%BENCHVIEW_PATH%\measurement.py" xunit "Perf-%BENCHNAME%.xml" --better desc --drop-first-value --append
+  )
   REM Save off the results to the root directory for recovery later in Jenkins
   xcopy Perf-%BENCHNAME%*.xml %CORECLR_REPO%\
   xcopy Perf-%BENCHNAME%*.etl %CORECLR_REPO%\
@@ -117,6 +125,11 @@ shift
 shift
 goto :ARGLOOP
 )
+IF /I [%1] == [-scenarioTest] (
+set IS_SCENARIO_TEST=1
+shift
+goto :ARGLOOP
+)
 IF /I [%1] == [-runtype] (
 set RUN_TYPE=%2
 shift
@@ -166,7 +179,7 @@ goto :USAGE
 goto :SETUP
 
 :USAGE
-echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-uploadToBenchview] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^>
+echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-uploadToBenchview] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^> [-scenarioTest]
 
 echo For the path to the tests you can pass a parent directory and the script will grovel for
 echo all tests in subdirectories and run them.
@@ -176,5 +189,6 @@ echo -uploadtoBenchview is used to specify a path to the Benchview tooling and w
 echo set we will upload the results of the tests to the coreclr container in benchviewupload.
 echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for
 echo PRs.
+echo -scenarioTest should be included if you are running a scenario benchmark.
 
 goto :EOF

diff --git a/tests/src/Common/PerfHarness/project.json b/tests/src/Common/PerfHarness/project.json
index d920190d119eb3820095f2e0921cf72102c4ca31..abbd7d9d9c2d3acb62db8b44406b78b9caee6bd3 100644 (file)
--- a/tests/src/Common/PerfHarness/project.json
+++ b/tests/src/Common/PerfHarness/project.json
@@ -12,7 +12,7 @@
           "type": "platform",
           "version": "1.1.0"
         },
-        "xunit.performance.api": "1.0.0-alpha-build0049"
+        "xunit.performance.api": "1.0.0-beta-build0002"
       }
     }
   }

diff --git a/tests/src/Common/external/project.json b/tests/src/Common/external/project.json
index 63cdc0425478d631b84b1ead69202884bcf1759b..de6b9f73668b8fd193b6f500e76c69ef6a587f7d 100644 (file)
--- a/tests/src/Common/external/project.json
+++ b/tests/src/Common/external/project.json
@@ -1,10 +1,10 @@
 {
   "dependencies": {
     "Microsoft.CodeAnalysis.Compilers": "1.1.1",
-    "xunit.performance.api": "1.0.0-alpha-build0049",
-    "xunit.performance.core": "1.0.0-alpha-build0049",
-    "xunit.performance.execution": "1.0.0-alpha-build0049",
-    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "xunit.performance.api": "1.0.0-beta-build0002",
+    "xunit.performance.core": "1.0.0-beta-build0002",
+    "xunit.performance.execution": "1.0.0-beta-build0002",
+    "xunit.performance.metrics": "1.0.0-beta-build0002",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Newtonsoft.Json": "9.0.1",
     "xunit": "2.2.0-beta2-build3300",
@@ -23,4 +23,4 @@
     "win7-x86": {},
     "win7-x64": {}
   }
-}
\ No newline at end of file
+}

diff --git a/tests/src/JIT/config/benchmark+roslyn/project.json b/tests/src/JIT/config/benchmark+roslyn/project.json
index 73a1d098eb1709f68ec90776ea2475122516ce2e..272b6aabf3affe20911ca5fd8df3a6edce3e4686 100644 (file)
--- a/tests/src/JIT/config/benchmark+roslyn/project.json
+++ b/tests/src/JIT/config/benchmark+roslyn/project.json
@@ -1,10 +1,10 @@
 {
   "dependencies": {
     "Microsoft.CodeAnalysis.Compilers": "1.1.1",
-    "xunit.performance.api": "1.0.0-alpha-build0049",
-    "xunit.performance.core": "1.0.0-alpha-build0049",
-    "xunit.performance.execution": "1.0.0-alpha-build0049",
-    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "xunit.performance.api": "1.0.0-beta-build0002",
+    "xunit.performance.core": "1.0.0-beta-build0002",
+    "xunit.performance.execution": "1.0.0-beta-build0002",
+    "xunit.performance.metrics": "1.0.0-beta-build0002",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview1-25214-03",
     "System.Console": "4.4.0-beta-24913-02",

diff --git a/tests/src/JIT/config/benchmark+serialize/project.json b/tests/src/JIT/config/benchmark+serialize/project.json
index c168170861921f74d17b5c2963880fe4b71dd341..d5cc9f5f7d37acbb28c1320efb9dfe9e262a5a2d 100644 (file)
--- a/tests/src/JIT/config/benchmark+serialize/project.json
+++ b/tests/src/JIT/config/benchmark+serialize/project.json
@@ -1,9 +1,9 @@
 {
   "dependencies": {
-    "xunit.performance.api": "1.0.0-alpha-build0049",
-    "xunit.performance.core": "1.0.0-alpha-build0049",
-    "xunit.performance.execution": "1.0.0-alpha-build0049",
-    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "xunit.performance.api": "1.0.0-beta-build0002",
+    "xunit.performance.core": "1.0.0-beta-build0002",
+    "xunit.performance.execution": "1.0.0-beta-build0002",
+    "xunit.performance.metrics": "1.0.0-beta-build0002",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview1-25214-03",
     "Newtonsoft.Json": "7.0.1",

diff --git a/tests/src/JIT/config/benchmark/project.json b/tests/src/JIT/config/benchmark/project.json
index 73aeca1930e8d21b91c76ac0c83b48bbd5851793..3b218fc2038ef6182aa0de6a6678aedf3fde7a6b 100644 (file)
--- a/tests/src/JIT/config/benchmark/project.json
+++ b/tests/src/JIT/config/benchmark/project.json
@@ -1,9 +1,9 @@
 {
   "dependencies": {
-    "xunit.performance.api": "1.0.0-alpha-build0049",
-    "xunit.performance.core": "1.0.0-alpha-build0049",
-    "xunit.performance.execution": "1.0.0-alpha-build0049",
-    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "xunit.performance.api": "1.0.0-beta-build0002",
+    "xunit.performance.core": "1.0.0-beta-build0002",
+    "xunit.performance.execution": "1.0.0-beta-build0002",
+    "xunit.performance.metrics": "1.0.0-beta-build0002",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview1-25214-03",
     "System.Collections.NonGeneric": "4.4.0-beta-24913-02",

diff --git a/tests/src/NuGet.Config b/tests/src/NuGet.Config
index 51701f05f607890c4f6fc09a7796e2f65b45afec..0922300baf79a253a0a47ee3577f98eb74c69618 100644 (file)
--- a/tests/src/NuGet.Config
+++ b/tests/src/NuGet.Config
@@ -7,7 +7,7 @@
     <clear/>
     <add key="myget.org dotnet-core" value="https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
     <add key="myget.org nugetbuild" value="https://www.myget.org/F/nugetbuild/api/v3/index.json" />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" protocolVersion="3" />
   </packageSources>
   <activePackageSource>
     <add key="All" value="(Aggregate source)" />

diff --git a/tests/src/performance/project.json b/tests/src/performance/project.json
index b1175d6330aa5721c7893b16a86f65ca835880db..c136c19a34b6d6b4b5b0488998bd171c9725f16f 100644 (file)
--- a/tests/src/performance/project.json
+++ b/tests/src/performance/project.json
@@ -1,9 +1,9 @@
 {
   "dependencies": {
-    "xunit.performance.api": "1.0.0-alpha-build0049",
-    "xunit.performance.core": "1.0.0-alpha-build0049",
-    "xunit.performance.execution": "1.0.0-alpha-build0049",
-    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "xunit.performance.api": "1.0.0-beta-build0002",
+    "xunit.performance.core": "1.0.0-beta-build0002",
+    "xunit.performance.execution": "1.0.0-beta-build0002",
+    "xunit.performance.metrics": "1.0.0-beta-build0002",
     "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-preview1-25214-03",
     "System.Collections.NonGeneric": "4.4.0-beta-24913-02",
@@ -17,6 +17,7 @@
     "System.Reflection.TypeExtensions": "4.4.0-preview1-25214-03",
     "System.Runtime": "4.4.0-beta-24913-02",
     "System.Runtime.Extensions": "4.4.0-beta-24913-02",
+    "System.Runtime.Loader": "4.0.0",
     "System.Runtime.Numerics": "4.4.0-beta-24913-02",
     "System.Text.RegularExpressions": "4.4.0-beta-24913-02",
     "System.Threading": "4.4.0-beta-24913-02",
@@ -31,7 +32,7 @@
     "xunit.runner.utility": "2.2.0-beta2-build3300"
   },
   "frameworks": {
-    "netstandard1.4": {
+    "netstandard1.5": {
       "imports": [
         "dnxcore50",
         "portable-net45+win8"