assert osGroup != null : "Could not find os group for ${os}"
return osGroupMap[os]
}
+// Setup perflab tests runs
+[true, false].each { isPR ->
+ ['Windows_NT'].each { os ->
+ ['x64'].each { architecture ->
+ def configuration = 'Release'
+ def runType = isPR ? 'private' : 'rolling'
+ def benchViewName = isPR ? 'coreclr private %ghprbPullTitle%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
+ def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}", isPR)) {
+ // Set the label.
+ label('windows_clr_perf')
+ wrappers {
+ credentialsBinding {
+ string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+ }
+ }
+
+ steps {
+ // Batch
+
+ batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
+ //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+ //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+ batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
+ "py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py --name " + "\"" + benchViewName + "\"" + " --user " + "\"dotnet-bot@microsoft.com\"\n" +
+ "py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\build.py git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type " + runType)
+ batchFile("py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
+ batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
+ batchFile("tests\\runtest.cmd release ${architecture} GenerateLayoutOnly")
+ batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\performance\\perflab\\Perflab -library -uploadToBenchview C:\\Tools\\Microsoft.Benchview.JSONFormat\\tools -runtype " + runType)
+ }
+ }
+
+ // Save machinedata.json to /artifact/bin/ Jenkins dir
+ def archiveSettings = new ArchivalSettings()
+ archiveSettings.addFiles('sandbox\\perf-*.xml')
+ Utilities.addArchival(newJob, archiveSettings)
+
+ Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+
+ if (isPR) {
+ TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+ builder.setGithubContext("${os} CoreCLR Perf Tests")
+ builder.triggerOnlyOnComment()
+ builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
+ builder.triggerForBranch(branch)
+ builder.emitTrigger(newJob)
+ }
+ else {
+ // Set a push trigger
+ TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+ builder.emitTrigger(newJob)
+ }
+ }
+ }
+}
[true, false].each { isPR ->
['Windows_NT'].each { os ->
def architecture = 'x64'
def configuration = 'Release'
def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
+
// Set the label.
label('performance')
steps {
// Batch
- batchFile("C:\\tools\\nuget install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
- batchFile("python C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
+ batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
+ batchFile("py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
batchFile("tests\\runtest.cmd release ${architecture}")
- batchFile("tests\\scripts\\run-xunit-perf.cmd")
+ batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.x64.Release\\Jit\\Performance\\CodeQuality")
}
}
}
}
- Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM\92s for now.
+ Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM's for now.
// Save machinedata.json to /artifact/bin/ Jenkins dir
def archiveSettings = new ArchivalSettings()
builder.emitTrigger(newJob)
}
} // os
-} // isPR
+} // isPR
\ No newline at end of file
@setlocal
@echo off
-set HERE=%CD%
+rem Set defaults for the file extension, architecture and configuration
set CORECLR_REPO=%CD%
-set CORECLR_OVERLAY=%CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Tests\Core_Root
-set CORECLR_PERF=%CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Jit\Performance\CodeQuality
-set RUNLOG=%HERE%\bin\Logs\perfrun.log
+set TEST_FILE_EXT=exe
+set TEST_ARCH=x64
+set TEST_CONFIG=Release
+
+goto :ARGLOOP
+
+:SETUP
+
+set CORECLR_OVERLAY=%CORECLR_REPO%\bin\tests\Windows_NT.%TEST_ARCH%.%TEST_CONFIG%\Tests\Core_Root
+set RUNLOG=%CORECLR_REPO%\bin\Logs\perfrun.log
if NOT EXIST %CORECLR_OVERLAY% (
echo Can't find test overlay directory '%CORECLR_OVERLAY%'
exit /B 1
)
-:SETUP
-
@echo --- setting up sandbox
rd /s /q sandbox
@rem xunit and perf
xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.runner.Windows\1.0.0-alpha-build0040\tools\* . > %RUNLOG%
-xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.analysis\1.0.0-alpha-build0040\tools\* . > %RUNLOG%
-xcopy /sy %CORECLR_REPO%\packages\xunit.console.netcore\1.0.2-prerelease-00101\runtimes\any\native\* . > %RUNLOG%
-xcopy /sy %CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\* . > %RUNLOG%
+xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.analysis\1.0.0-alpha-build0040\tools\* . >> %RUNLOG%
+xcopy /sy %CORECLR_REPO%\packages\xunit.console.netcore\1.0.2-prerelease-00177\runtimes\any\native\* . >> %RUNLOG%
+xcopy /sy %CORECLR_REPO%\bin\tests\Windows_NT.%TEST_ARCH%.%TEST_CONFIG%\Tests\Core_Root\* . >> %RUNLOG%
@rem find and stage the tests
-
-for /R %CORECLR_PERF% %%T in (*.exe) do (
+for /R %CORECLR_PERF% %%T in (*.%TEST_FILE_EXT%) do (
call :DOIT %%T
)
goto :EOF
:DOIT
-
set BENCHNAME=%~n1
set PERFOUT=perf-%BENCHNAME%
set XMLOUT=%PERFOUT%-summary.xml
echo --- Running %BENCHNAME%
-xcopy /s %1 . > %RUNLOG%
+xcopy /s %1 . >> %RUNLOG%
-set CORE_ROOT=%HERE%\sandbox
+set CORE_ROOT=%CORECLR_REPO%\sandbox
-xunit.performance.run.exe %BENCHNAME%.exe -runner xunit.console.netcore.exe -runnerhost corerun.exe -verbose -runid %PERFOUT% > %BENCHNAME%.out
+xunit.performance.run.exe %BENCHNAME%.%TEST_FILE_EXT% -runner xunit.console.netcore.exe -runnerhost corerun.exe -verbose -runid %PERFOUT% > %BENCHNAME%.out
xunit.performance.analysis.exe %PERFOUT%.xml -xml %XMLOUT% > %BENCHNAME%-analysis.out
-type %XMLOUT% | findstr "test name"
-type %XMLOUT% | findstr Duration
-type %XMLOUT% | findstr InstRetired
+@rem optionally upload results to benchview
+if not [%BENCHVIEW_PATH%] == [] (
+ py %BENCHVIEW_PATH%\measurement.py xunit perf-Perflab.xml --better desc --drop-first-value
+ py %BENCHVIEW_PATH%\submission.py measurement.json ^
+ --build ..\build.json ^
+ --machine-data ..\machinedata.json ^
+ --metadata ..\submission-metadata.json ^
+ --group "CoreCLR" ^
+ --type "%RUN_TYPE%" ^
+ --config-name "%TEST_CONFIG%" ^
+ --config Configuration "%TEST_CONFIG%" ^
+ --config OS "Windows_NT" ^
+ --arch "%TEST_ARCH%" ^
+ --machinepool "PerfSnake"
+ py %BENCHVIEW_PATH%\upload.py submission.json --container coreclr
+) else (
+ type %XMLOUT% | findstr "test name"
+ type %XMLOUT% | findstr Duration
+ type %XMLOUT% | findstr InstRetired
+)
+
+goto :EOF
+
+:ARGLOOP
+IF /I [%1] == [-testBinLoc] (
+set CORECLR_PERF=%CORECLR_REPO%\%2
+shift
+shift
+goto :ARGLOOP
+)
+IF /I [%1] == [-runtype] (
+set RUN_TYPE=%2
+shift
+shift
+goto :ARGLOOP
+)
+IF /I [%1] == [-library] (
+set TEST_FILE_EXT=dll
+shift
+goto :ARGLOOP
+)
+IF /I [%1] == [-uploadtobenchview] (
+set BENCHVIEW_PATH=%2
+shift
+shift
+goto :ARGLOOP
+)
+IF /I [%1] == [-arch] (
+set TEST_ARCH=%2
+shift
+shift
+goto :ARGLOOP
+)
+IF /I [%1] == [-configuration] (
+set TEST_CONFIG=%2
+shift
+shift
+goto :ARGLOOP
+)
+if /I [%1] == [-?] (
+goto :USAGE
+)
+if /I [%1] == [-help] (
+goto :USAGE
+)
+
+if [%CORECLR_PERF%] == [] (
+goto :USAGE
+)
+
+goto :SETUP
+
+:USAGE
+echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-uploadToBenchview] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^>
+
+echo For the path to the tests you can pass a parent directory and the script will grovel for
+echo all tests in subdirectories and run them.
+echo The library flag denotes whether the tests are build as libraries (.dll) or an executable (.exe)
+echo Architecture defaults to x64 and configuration defaults to release.
+echo -uploadtoBenchview is used to specify a path to the Benchview tooling and when this flag is
+echo set we will upload the results of the tests to the coreclr container in benchviewupload.
+echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for
+echo PRs.
projects / platform / upstream / dotnet / runtime.git / commitdiff