--- /dev/null
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+ <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+ <Python>python3</Python>
+ <HelixPreCommands>$(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk</HelixPreCommands>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
+ <PayloadDirectory>%(Identity)</PayloadDirectory>
+ </HelixCorrelationPayload>
+ </ItemGroup>
+
+ <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
+ <ScenarioDirectory>%HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\</ScenarioDirectory>
+ <BlazorDirectory>$(ScenarioDirectory)blazor\</BlazorDirectory>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+ <ScenarioDirectory>$HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/</ScenarioDirectory>
+ <BlazorDirectory>$(ScenarioDirectory)blazor/</BlazorDirectory>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <HelixWorkItem Include="SOD - New Blazor Template - Publish">
+ <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+ <PreCommands>cd $(BlazorDirectory);$(Python) pre.py publish --msbuild %27/p:_TrimmerDumpDependencies=true%27 --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog %27./traces/blazor_publish.binlog%27</PreCommands>
+ <Command>$(Python) test.py sod --scenario-name "%(Identity)"</Command>
+ <PostCommands>$(Python) post.py</PostCommands>
+ </HelixWorkItem>
+ </ItemGroup>
+</Project>
\ No newline at end of file
<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+ <ItemGroup>
+ <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
+ <PayloadDirectory>%(Identity)</PayloadDirectory>
+ </HelixCorrelationPayload>
+ </ItemGroup>
+
+ <!--
+ Crossgen and Crossgen2 Scenario WorkItems
+ -->
<PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
<Python>py -3</Python>
- <HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%</HelixPreCommands>
- <ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
- <BaseDirectory>$HELIX_CORRELATION_PAYLOAD</BaseDirectory>
- <PerformanceDirectory>$(BaseDirectory)/performance</PerformanceDirectory>
+ <HelixPreCommands>$(HelixPreCommands)</HelixPreCommands>
+ <CoreRoot>%HELIX_CORRELATION_PAYLOAD%\Core_Root</CoreRoot>
+ <ScenarioDirectory>%HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\</ScenarioDirectory>
+ <CrossgenDirectory>$(ScenarioDirectory)crossgen\</CrossgenDirectory>
+ <Crossgen2Directory>$(ScenarioDirectory)crossgen2\</Crossgen2Directory>
</PropertyGroup>
-
- <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
- <WorkItemCommand>$(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj)</WorkItemCommand>
- <CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
+ <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
<Python>python3</Python>
- <CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
- <HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
- <ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
- <BaselineArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
- <ResultsComparer>$(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj</ResultsComparer>
- <DotnetExe>$(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet</DotnetExe>
- <Percent>%25</Percent>
- <XMLResults>$HELIX_WORKITEM_ROOT/testResults.xml</XMLResults>
- </PropertyGroup>
+ <HelixPreCommands>$(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update</HelixPreCommands>
+ <CoreRoot>$HELIX_CORRELATION_PAYLOAD/Core_Root</CoreRoot>
+ <ScenarioDirectory>$HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/</ScenarioDirectory>
+ <CrossgenDirectory>$(ScenarioDirectory)crossgen/</CrossgenDirectory>
+ <Crossgen2Directory>$(ScenarioDirectory)crossgen2/</Crossgen2Directory>
+ </PropertyGroup>
<ItemGroup>
- <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
- <PayloadDirectory>%(Identity)</PayloadDirectory>
- </HelixCorrelationPayload>
+ <SingleAssembly Include="System.Private.Xml.dll"/>
+ <SingleAssembly Include="System.Linq.Expressions.dll"/>
+ <SingleAssembly Include="Microsoft.CodeAnalysis.VisualBasic.dll"/>
+ <SingleAssembly Include="Microsoft.CodeAnalysis.CSharp.dll"/>
+ <SingleAssembly Include="System.Private.CoreLib.dll"/>
+ </ItemGroup>
+ <ItemGroup>
+ <Composite Include="framework-r2r.dll.rsp"/>
</ItemGroup>
- <ItemGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
- <HelixWorkItem Include="Crossgen System.Private.Xml.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
- </HelixWorkItem>
- <HelixWorkItem Include="Crossgen System.Linq.Expressions.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
- </HelixWorkItem>
- <HelixWorkItem Include="Crossgen Microsoft.CodeAnalysis.VisualBasic.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
- </HelixWorkItem>
- <HelixWorkItem Include="Crossgen Microsoft.CodeAnalysis.CSharp.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
- </HelixWorkItem>
- <HelixWorkItem Include="Crossgen System.Private.CoreLib.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
- </HelixWorkItem>
+ <ItemGroup>
+ <CrossgenWorkItem Include="@(SingleAssembly)">
+ <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+ <Command>$(Python) $(CrossgenDirectory)test.py crossgen --core-root $(CoreRoot) --test-name %(Identity)</Command>
+ </CrossgenWorkItem>
</ItemGroup>
- <ItemGroup Condition="'$(AGENT_OS)' == 'Windows_NT' and '$(Architecture)' == 'x64'">
- <HelixWorkItem Include="Crossgen2 System.Private.Xml.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
- </HelixWorkItem>
- <HelixWorkItem Include="Crossgen2 System.Linq.Expressions.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
- </HelixWorkItem>
- <HelixWorkItem Include="Crossgen2 Microsoft.CodeAnalysis.VisualBasic.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
- </HelixWorkItem>
- <HelixWorkItem Include="Crossgen2 Microsoft.CodeAnalysis.CSharp.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
+ <ItemGroup>
+ <Crossgen2WorkItem Include="@(SingleAssembly)">
+ <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+ <Command>$(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity)</Command>
+ </Crossgen2WorkItem>
+ </ItemGroup>
+
+ <ItemGroup>
+ <!-- Enable crossgen tests on Windows x64 and Windows x86 -->
+ <HelixWorkItem Include="@(CrossgenWorkItem -> 'Crossgen %(Identity)')" Condition="'$(AGENT_OS)' == 'Windows_NT'">
+ <Timeout>4:00</Timeout>
</HelixWorkItem>
- <HelixWorkItem Include="Crossgen2 System.Private.CoreLib.dll">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
+ <!-- Enable crossgen2 tests on Windows x64 and Linux x64 -->
+ <HelixWorkItem Include="@(Crossgen2WorkItem -> 'Crossgen2 %(Identity)')" Condition="'$(Architecture)' == 'x64'">
+ <Timeout>4:00</Timeout>
</HelixWorkItem>
- <HelixWorkItem Include="Crossgen2 Composite Framework R2R">
- <PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
- <Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
+ <HelixWorkItem Include="Crossgen2 Composite Framework R2R" Condition="'$(Architecture)' == 'x64'">
+ <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+ <Command>$(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp</Command>
<Timeout>1:00</Timeout>
</HelixWorkItem>
</ItemGroup>
</PropertyGroup>
<PropertyGroup Condition="'$(MonoDotnet)' == 'true' and '$(AGENT_OS)' == 'Windows_NT'">
- <CoreRunArgument>--corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe</CoreRunArgument>
+ <CoreRunArgument>--corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\6.0.0\corerun.exe</CoreRunArgument>
</PropertyGroup>
<PropertyGroup Condition="'$(MonoDotnet)' == 'true' and '$(AGENT_OS)' != 'Windows_NT'">
- <CoreRunArgument>--corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun</CoreRunArgument>
+ <CoreRunArgument>--corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun</CoreRunArgument>
</PropertyGroup>
<PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
using_mono=false
wasm_runtime_loc=
using_wasm=false
+use_latest_dotnet=false
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
configurations=$2
shift 2
;;
- --help)
+ --latestdotnet)
+ use_latest_dotnet=true
+ shift 1
+ ;;
+ *)
echo "Common settings:"
echo " --corerootdirectory <value> Directory where Core_Root exists, if running perf testing with --corerun"
echo " --architecture <value> Architecture of the testing being run"
echo " --internal If the benchmarks are running as an official job."
echo " --monodotnet Pass the path to the mono dotnet for mono performance testing."
echo " --wasm Path to the unpacked wasm runtime pack."
+ echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json "
echo ""
exit 0
;;
fi
fi
-if [[ "$mono_dotnet" != "" ]]; then
+if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
+ extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono"
fi
if [[ "$wasm_runtime_loc" != "" ]]; then
- configurations="CompilationMode=wasm RunKind=micro"
- extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM"
+ configurations="CompilationMode=wasm RunKind=$kind"
+ extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM NoMono"
fi
-if [[ "$monointerpreter" == "true" ]]; then
- extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter"
+if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
+ extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
fi
common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
-# Get the tools section from the global.json.
-# This grabs the LKG version number of dotnet and passes it to our scripts
-dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'`
-setup_arguments="--dotnet-versions $dotnet_version $setup_arguments"
-
+if [[ "$use_latest_dotnet" = false ]]; then
+ # Get the tools section from the global.json.
+ # This grabs the LKG version number of dotnet and passes it to our scripts
+ dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'`
+ setup_arguments="--dotnet-versions $dotnet_version $setup_arguments"
+fi
if [[ "$run_from_perf_repo" = true ]]; then
payload_directory=
Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false
Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false
Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false
-Write-PipelineSetVariable -name "Python" -value "$python3" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Python" -value "python3" -is_multi_job_variable false
Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false
Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false
Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false
runtimeType: wasm
codeGenType: 'wasm'
projectFile: microbenchmarks.proj
- runKind: micro # is this supposed to be micro_mono or micro_wasm?
+ runKind: micro
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
buildConfig: release
runtimeFlavor: coreclr
platforms:
- #- Linux_x64
+ - Linux_x64
- Windows_NT_x64
- Windows_NT_x86
jobParameters:
- IsInternal: ''
- HelixApiAccessToken: ''
-
- # run machine-setup and set PYTHONPATH
+ - HelixPreCommands: ''
+ - AdditionalHelixPreCommands: ''
+ # run machine-setup and set PYTHONPATH for both public and private jobs
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- HelixPreCommands: 'call %HELIX_WORKITEM_PAYLOAD%\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%'
- ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
# extra private job settings
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- - HelixPreCommands: py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install --user azure.storage.blob==12.0.0 --force-reinstall;py -3 -m pip install --user azure.storage.queue==12.0.0 --force-reinstall;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)";$(HelixPreCommands)
- - IsInternal: -Internal
+ - AdditionalHelixPreCommands: py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install --user azure.storage.blob==12.0.0 --force-reinstall;py -3 -m pip install --user azure.storage.queue==12.0.0 --force-reinstall;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken);"
+ - IsInternal: -Internal
- ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
- - HelixPreCommands: sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/Scripts/activate;export PYTHONPATH=;pip3 install --user azure.storage.blob==12.0.0 --force-reinstall;pip3 install --user azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)";$(HelixPreCommands)
+ - AdditionalHelixPreCommands: sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/Scripts/activate;export PYTHONPATH=;pip3 install --user azure.storage.blob==12.0.0 --force-reinstall;pip3 install --user azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux);"
- IsInternal: --internal
- group: DotNet-HelixApi-Access
- group: dotnet-benchview
HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)'
HelixAccessToken: $(HelixApiAccessToken)
HelixTargetQueues: $(Queue)
- HelixPreCommands: $(HelixPreCommands)
+ HelixPreCommands: '$(AdditionalHelixPreCommands)$(HelixPreCommands)' # $(HelixPreCommands) should follow $(AdditionalHelixPreCommands) because PYTHONPATH is cleared by the former
Creator: $(Creator)
WorkItemTimeout: 4:00 # 4 hours
WorkItemDirectory: '$(WorkItemDirectory)' # contains scenario tools, shared python scripts, dotnet tool