Add smoketest for performance tests

This adds a run that will fire on the set of Azure VMs and will only run
for one interation.  This along with the changes to fail a performance
run if a test fails will allow us to start gating checkins on the
performance tests still running.

Commit migrated from https://github.com/dotnet/coreclr/commit/6ed8e17a41891b9d149cbe423eea5f779f8591d9

diff --git a/src/coreclr/perf.groovy b/src/coreclr/perf.groovy
index 1eb115d..b490219 100644 (file)
--- a/src/coreclr/perf.groovy
+++ b/src/coreclr/perf.groovy
@@ -28,78 +28,110 @@ def static getOSGroup(def os) {
 [true, false].each { isPR ->
     ['Windows_NT'].each { os ->
         ['x64', 'x86'].each { arch ->
-            def architecture = arch
+            [true, false].each { isSmoketest ->
+                def architecture = arch
+                def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_smoketest" : "perf_perflab_${os}_${arch}"
 
-            def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}_${arch}", isPR)) {
-                // Set the label.
-                label('windows_clr_perf')
-                wrappers {
-                    credentialsBinding {
-                        string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
-                    }
+                if (arch == 'x86jit32')
+                {
+                    architecture = 'x86'
+                    testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
+                }
+                else if (arch == 'x86')
+                {
+                    testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
                 }
 
-            if (isPR)
-            {
-                parameters
+                def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
+                    // Set the label.
+                    label('windows_clr_perf')
+                    wrappers {
+                        credentialsBinding {
+                            string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+                        }
+                    }
+
+                if (isPR)
                 {
-                    stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
+                    parameters
+                    {
+                        stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
+                    }
                 }
-            }
-            def configuration = 'Release'
-            def runType = isPR ? 'private' : 'rolling'
-            def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
-                
-                steps {
-                    // Batch
+                if (isSmoketest)
+                {
+                    parameters
+                    {
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to one.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
+                        stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to one.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
+                    }
+                }
+                def configuration = 'Release'
+                def runType = isPR ? 'private' : 'rolling'
+                def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
+                def uploadString = isSmoketest ? '' : '-uploadToBenchview'
+                    
+                    steps {
+                        // Batch
 
-                    batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
-                    batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
-                    //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
-                    //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
-                    batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
-                    "set BENCHVIEWNAME=${benchViewName}\n" +
-                    "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
-                    "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
-                    "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
-                    batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
-                    batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
+                        batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+                        batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
+                        batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+                        //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+                        //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+                        batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
+                        "set BENCHVIEWNAME=${benchViewName}\n" +
+                        "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
+                        "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
+                        "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
+                        batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
+                        batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 
-                    batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
+                        if (arch == 'x86jit32')
+                        {
+                            // Download package and copy compatjit into Core_Root
+                            batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
+                            "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
+                        }
 
-                    batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
-                    batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
-                }
-            }
+                        batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 
-            // Save machinedata.json to /artifact/bin/ Jenkins dir
-            def archiveSettings = new ArchivalSettings()
-            archiveSettings.addFiles('Perf-*.xml')
-            archiveSettings.addFiles('Perf-*.etl')
-            Utilities.addArchival(newJob, archiveSettings)
+                        batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}")
+                        batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}")
+                    }
+                }
+                
+                if (isSmoketest)
+                {
+                    Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
+                }
+                // Save machinedata.json to /artifact/bin/ Jenkins dir
+                def archiveSettings = new ArchivalSettings()
+                archiveSettings.addFiles('Perf-*.xml')
+                archiveSettings.addFiles('Perf-*.etl')
+                Utilities.addArchival(newJob, archiveSettings)
 
-            Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
-            
-            newJob.with {
-                wrappers {
-                    timeout {
-                        absolute(240)
+                Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+                
+                newJob.with {
+                    wrappers {
+                        timeout {
+                            absolute(240)
+                        }
                     }
                 }
-            }
-            
-            if (isPR) {
-                TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
-                builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
-                builder.triggerOnlyOnComment()
-                builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
-                builder.triggerForBranch(branch)
-                builder.emitTrigger(newJob)
-            }
-            else {
-                // Set a push trigger
-                TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
-                builder.emitTrigger(newJob)
+                
+                if (isPR) {
+                    TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+                    builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
+                    builder.triggerForBranch(branch)
+                    builder.emitTrigger(newJob)
+                }
+                else {
+                    // Set a push trigger
+                    TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+                    builder.emitTrigger(newJob)
+                }
             }
         }
     }

diff --git a/src/coreclr/tests/scripts/run-xunit-perf.cmd b/src/coreclr/tests/scripts/run-xunit-perf.cmd
index 80f8544..11279ae 100644 (file)
--- a/src/coreclr/tests/scripts/run-xunit-perf.cmd
+++ b/src/coreclr/tests/scripts/run-xunit-perf.cmd
@@ -5,6 +5,7 @@
 @echo off
 @if defined _echo echo on
 
+setlocal ENABLEDELAYEDEXPANSION
 setlocal
   set ERRORLEVEL=
   set BENCHVIEW_RUN_TYPE=local
@@ -15,14 +16,17 @@ setlocal
   set TEST_CONFIG=Release
   set IS_SCENARIO_TEST=
   set USAGE_DISPLAYED=
+  set SHOULD_UPLOAD_TO_BENCHVIEW=
+  set BENCHVIEW_PATH=
 
   call :parse_command_line_arguments %*
   if defined USAGE_DISPLAYED exit /b %ERRORLEVEL%
 
-  call :set_test_architecture || exit /b 1
-  call :verify_core_overlay   || exit /b 1
-  call :set_perf_run_log      || exit /b 1
-  call :setup_sandbox         || exit /b 1
+  call :set_test_architecture  || exit /b 1
+  call :verify_benchview_tools || exit /b 1
+  call :verify_core_overlay    || exit /b 1
+  call :set_perf_run_log       || exit /b 1
+  call :setup_sandbox          || exit /b 1
 
   call :run_cmd "%CORECLR_REPO%\Tools\dotnetcli\dotnet.exe" restore "%CORECLR_REPO%\tests\src\Common\PerfHarness\project.json"                                        || exit /b 1
   call :run_cmd "%CORECLR_REPO%\Tools\dotnetcli\dotnet.exe" publish "%CORECLR_REPO%\tests\src\Common\PerfHarness\project.json" -c Release -o "%CORECLR_REPO%\sandbox" || exit /b 1
@@ -92,7 +96,7 @@ setlocal
   )
 
   rem optionally generate results for benchview
-  if not [%BENCHVIEW_PATH%] == [] (
+  if exist "%BENCHVIEW_PATH%" (
     call :generate_results_for_benchview || exit /b 1
   ) else (
     type "%XMLOUT%" | findstr /i /c:"test name"
@@ -119,6 +123,11 @@ rem ****************************************************************************
     shift
     goto :parse_command_line_arguments
   )
+  IF /I [%~1] == [-uploadtobenchview] (
+    set SHOULD_UPLOAD_TO_BENCHVIEW=1
+    shift
+    goto :parse_command_line_arguments
+  )
   IF /I [%~1] == [-runtype] (
     set BENCHVIEW_RUN_TYPE=%~2
     shift
@@ -130,7 +139,7 @@ rem ****************************************************************************
     shift
     goto :parse_command_line_arguments
   )
-  IF /I [%~1] == [-uploadtobenchview] (
+  IF /I [%~1] == [-generatebenchviewdata] (
     set BENCHVIEW_PATH=%~2
     shift
     shift
@@ -175,7 +184,19 @@ rem   Sets the test architecture.
 rem ****************************************************************************
   set TEST_ARCH=%TEST_ARCHITECTURE%
   exit /b 0
-
+  
+:verify_benchview_tools
+rem ****************************************************************************
+rem   Verifies that the path to the benchview tools is correct.
+rem ****************************************************************************
+  if defined BENCHVIEW_PATH (
+    if not exist "%BENCHVIEW_PATH%" (
+      call :print_error BenchView path: "%BENCHVIEW_PATH%" was specified, but it does not exist.
+      exit /b 1
+    )
+  )
+  exit /b 0
+  
 :verify_core_overlay
 rem ****************************************************************************
 rem   Verify that the Core_Root folder exist.
@@ -253,10 +274,12 @@ setlocal
     exit /b 1
   )
 
-  call :run_cmd py.exe "%BENCHVIEW_PATH%\upload.py" submission.json --container coreclr
-  IF %ERRORLEVEL% NEQ 0 (
-    call :print_error Uploading to BenchView failed.
-    exit /b 1
+  if defined SHOULD_UPLOAD_TO_BENCHVIEW (
+    call :run_cmd py.exe "%BENCHVIEW_PATH%\upload.py" submission.json --container coreclr
+    IF !ERRORLEVEL! NEQ 0 (
+      call :print_error Uploading to BenchView failed.
+      exit /b 1
+    )
   )
   exit /b %ERRORLEVEL%
 
@@ -265,14 +288,15 @@ rem ****************************************************************************
 rem   Script's usage.
 rem ****************************************************************************
   set USAGE_DISPLAYED=1
-  echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-uploadToBenchview] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^> [-scenarioTest]
+  echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-generateBenchviewData] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^> [-scenarioTest]
   echo/
   echo For the path to the tests you can pass a parent directory and the script will grovel for
   echo all tests in subdirectories and run them.
   echo The library flag denotes whether the tests are build as libraries (.dll) or an executable (.exe)
   echo Architecture defaults to x64 and configuration defaults to release.
-  echo -uploadtoBenchview is used to specify a path to the Benchview tooling and when this flag is
-  echo set we will upload the results of the tests to the coreclr container in benchviewupload.
+  echo -generateBenchviewData is used to specify a path to the Benchview tooling and when this flag is
+  echo set we will generate the results for upload to benchview.
+  echo -uploadToBenchview If this flag is set the generated benchview test data will be uploaded.
   echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for
   echo PRs.
   echo -scenarioTest should be included if you are running a scenario benchmark.