From: Swaroop Sridhar Date: Fri, 14 Apr 2017 03:07:41 +0000 (-0700) Subject: Add scenario-based tests X-Git-Tag: accepted/tizen/base/20180629.140029~1394 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=81c550179aef561fd6578109ff65185d1daafe8b;p=platform%2Fupstream%2Fcoreclr.git Add scenario-based tests Modified Xunit runner to run Scenario Benchmarks Updating xunit-performance packages --- diff --git a/tests/scripts/run-xunit-perf.cmd b/tests/scripts/run-xunit-perf.cmd index 0d657c6..3645632 100644 --- a/tests/scripts/run-xunit-perf.cmd +++ b/tests/scripts/run-xunit-perf.cmd @@ -12,6 +12,7 @@ set TEST_FILE_EXT=exe set TEST_ARCH=x64 set TEST_ARCHITECTURE=x64 set TEST_CONFIG=Release +set IS_SCENARIO_TEST=0 goto :ARGLOOP @@ -34,7 +35,6 @@ if NOT EXIST %CORECLR_OVERLAY% ( ) @echo --- setting up sandbox - if exist sandbox rd /s /q sandbox if exist sandbox echo ERROR: Failed to remove the sandbox folder& exit /b 1 if not exist sandbox mkdir sandbox @@ -94,11 +94,19 @@ if DEFINED TEST_ENV ( ) ) -corerun.exe PerfHarness.dll %CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT% --perf:runid Perf > %BENCHNAME%.out +if [%IS_SCENARIO_TEST%] == [1] ( + corerun.exe %CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT% --perf:runid Perf > %BENCHNAME%.out +) else ( + corerun.exe PerfHarness.dll %CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT% --perf:runid Perf > %BENCHNAME%.out +) @rem optionally generate results for benchview if not [%BENCHVIEW_PATH%] == [] ( - py "%BENCHVIEW_PATH%\measurement.py" xunit "Perf-%BENCHNAME%.xml" --better desc --drop-first-value --append + if [%IS_SCENARIO_TEST%] == [1] ( + py "%BENCHVIEW_PATH%\measurement.py" xunitscenario "Perf-%BENCHNAME%.xml" --better desc --drop-first-value --append + ) else ( + py "%BENCHVIEW_PATH%\measurement.py" xunit "Perf-%BENCHNAME%.xml" --better desc --drop-first-value --append + ) REM Save off the results to the root directory for recovery later in Jenkins xcopy Perf-%BENCHNAME%*.xml %CORECLR_REPO%\ xcopy Perf-%BENCHNAME%*.etl %CORECLR_REPO%\ @@ -117,6 +125,11 @@ shift shift goto :ARGLOOP ) +IF /I [%1] == [-scenarioTest] ( +set IS_SCENARIO_TEST=1 +shift +goto :ARGLOOP +) IF /I [%1] == [-runtype] ( set RUN_TYPE=%2 shift @@ -166,7 +179,7 @@ goto :USAGE goto :SETUP :USAGE -echo run-xunit-perf.cmd -testBinLoc ^ [-library] [-arch] ^ [-configuration] ^ [-uploadToBenchview] ^ [-runtype] ^ +echo run-xunit-perf.cmd -testBinLoc ^ [-library] [-arch] ^ [-configuration] ^ [-uploadToBenchview] ^ [-runtype] ^ [-scenarioTest] echo For the path to the tests you can pass a parent directory and the script will grovel for echo all tests in subdirectories and run them. @@ -176,5 +189,6 @@ echo -uploadtoBenchview is used to specify a path to the Benchview tooling and w echo set we will upload the results of the tests to the coreclr container in benchviewupload. echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for echo PRs. +echo -scenarioTest should be included if you are running a scenario benchmark. goto :EOF diff --git a/tests/src/Common/PerfHarness/project.json b/tests/src/Common/PerfHarness/project.json index d920190..abbd7d9 100644 --- a/tests/src/Common/PerfHarness/project.json +++ b/tests/src/Common/PerfHarness/project.json @@ -12,7 +12,7 @@ "type": "platform", "version": "1.1.0" }, - "xunit.performance.api": "1.0.0-alpha-build0049" + "xunit.performance.api": "1.0.0-beta-build0002" } } } diff --git a/tests/src/Common/external/project.json b/tests/src/Common/external/project.json index 63cdc04..de6b9f7 100644 --- a/tests/src/Common/external/project.json +++ b/tests/src/Common/external/project.json @@ -1,10 +1,10 @@ { "dependencies": { "Microsoft.CodeAnalysis.Compilers": "1.1.1", - "xunit.performance.api": "1.0.0-alpha-build0049", - "xunit.performance.core": "1.0.0-alpha-build0049", - "xunit.performance.execution": "1.0.0-alpha-build0049", - "xunit.performance.metrics": "1.0.0-alpha-build0049", + "xunit.performance.api": "1.0.0-beta-build0002", + "xunit.performance.core": "1.0.0-beta-build0002", + "xunit.performance.execution": "1.0.0-beta-build0002", + "xunit.performance.metrics": "1.0.0-beta-build0002", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Newtonsoft.Json": "9.0.1", "xunit": "2.2.0-beta2-build3300", @@ -23,4 +23,4 @@ "win7-x86": {}, "win7-x64": {} } -} \ No newline at end of file +} diff --git a/tests/src/JIT/config/benchmark+roslyn/project.json b/tests/src/JIT/config/benchmark+roslyn/project.json index 73a1d09..272b6aa 100644 --- a/tests/src/JIT/config/benchmark+roslyn/project.json +++ b/tests/src/JIT/config/benchmark+roslyn/project.json @@ -1,10 +1,10 @@ { "dependencies": { "Microsoft.CodeAnalysis.Compilers": "1.1.1", - "xunit.performance.api": "1.0.0-alpha-build0049", - "xunit.performance.core": "1.0.0-alpha-build0049", - "xunit.performance.execution": "1.0.0-alpha-build0049", - "xunit.performance.metrics": "1.0.0-alpha-build0049", + "xunit.performance.api": "1.0.0-beta-build0002", + "xunit.performance.core": "1.0.0-beta-build0002", + "xunit.performance.execution": "1.0.0-beta-build0002", + "xunit.performance.metrics": "1.0.0-beta-build0002", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Microsoft.NETCore.Platforms": "2.0.0-preview1-25214-03", "System.Console": "4.4.0-beta-24913-02", diff --git a/tests/src/JIT/config/benchmark+serialize/project.json b/tests/src/JIT/config/benchmark+serialize/project.json index c168170..d5cc9f5 100644 --- a/tests/src/JIT/config/benchmark+serialize/project.json +++ b/tests/src/JIT/config/benchmark+serialize/project.json @@ -1,9 +1,9 @@ { "dependencies": { - "xunit.performance.api": "1.0.0-alpha-build0049", - "xunit.performance.core": "1.0.0-alpha-build0049", - "xunit.performance.execution": "1.0.0-alpha-build0049", - "xunit.performance.metrics": "1.0.0-alpha-build0049", + "xunit.performance.api": "1.0.0-beta-build0002", + "xunit.performance.core": "1.0.0-beta-build0002", + "xunit.performance.execution": "1.0.0-beta-build0002", + "xunit.performance.metrics": "1.0.0-beta-build0002", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Microsoft.NETCore.Platforms": "2.0.0-preview1-25214-03", "Newtonsoft.Json": "7.0.1", diff --git a/tests/src/JIT/config/benchmark/project.json b/tests/src/JIT/config/benchmark/project.json index 73aeca1..3b218fc 100644 --- a/tests/src/JIT/config/benchmark/project.json +++ b/tests/src/JIT/config/benchmark/project.json @@ -1,9 +1,9 @@ { "dependencies": { - "xunit.performance.api": "1.0.0-alpha-build0049", - "xunit.performance.core": "1.0.0-alpha-build0049", - "xunit.performance.execution": "1.0.0-alpha-build0049", - "xunit.performance.metrics": "1.0.0-alpha-build0049", + "xunit.performance.api": "1.0.0-beta-build0002", + "xunit.performance.core": "1.0.0-beta-build0002", + "xunit.performance.execution": "1.0.0-beta-build0002", + "xunit.performance.metrics": "1.0.0-beta-build0002", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Microsoft.NETCore.Platforms": "2.0.0-preview1-25214-03", "System.Collections.NonGeneric": "4.4.0-beta-24913-02", diff --git a/tests/src/NuGet.Config b/tests/src/NuGet.Config index 51701f0..0922300 100644 --- a/tests/src/NuGet.Config +++ b/tests/src/NuGet.Config @@ -7,7 +7,7 @@ - + diff --git a/tests/src/performance/project.json b/tests/src/performance/project.json index b1175d6..c136c19 100644 --- a/tests/src/performance/project.json +++ b/tests/src/performance/project.json @@ -1,9 +1,9 @@ { "dependencies": { - "xunit.performance.api": "1.0.0-alpha-build0049", - "xunit.performance.core": "1.0.0-alpha-build0049", - "xunit.performance.execution": "1.0.0-alpha-build0049", - "xunit.performance.metrics": "1.0.0-alpha-build0049", + "xunit.performance.api": "1.0.0-beta-build0002", + "xunit.performance.core": "1.0.0-beta-build0002", + "xunit.performance.execution": "1.0.0-beta-build0002", + "xunit.performance.metrics": "1.0.0-beta-build0002", "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental", "Microsoft.NETCore.Platforms": "2.0.0-preview1-25214-03", "System.Collections.NonGeneric": "4.4.0-beta-24913-02", @@ -17,6 +17,7 @@ "System.Reflection.TypeExtensions": "4.4.0-preview1-25214-03", "System.Runtime": "4.4.0-beta-24913-02", "System.Runtime.Extensions": "4.4.0-beta-24913-02", + "System.Runtime.Loader": "4.0.0", "System.Runtime.Numerics": "4.4.0-beta-24913-02", "System.Text.RegularExpressions": "4.4.0-beta-24913-02", "System.Threading": "4.4.0-beta-24913-02", @@ -31,7 +32,7 @@ "xunit.runner.utility": "2.2.0-beta2-build3300" }, "frameworks": { - "netstandard1.4": { + "netstandard1.5": { "imports": [ "dnxcore50", "portable-net45+win8"