* Update dependencies from https://github.com/dotnet/arcade build
20190822.24
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19422.24
- Microsoft.DotNet.Build.Tasks.Feed - 2.2.0-beta.19422.24
- Microsoft.DotNet.Build.Tasks.Packaging - 1.0.0-beta.19422.24
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19422.24
* Update dependencies from https://github.com/dotnet/arcade build
20190823.6
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19423.6
- Microsoft.DotNet.Build.Tasks.Feed - 2.2.0-beta.19423.6
- Microsoft.DotNet.Build.Tasks.Packaging - 1.0.0-beta.19423.6
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19423.6
* Update dependencies from https://github.com/dotnet/arcade build
20190824.1
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19424.1
- Microsoft.DotNet.Build.Tasks.Feed - 2.2.0-beta.19424.1
- Microsoft.DotNet.Build.Tasks.Packaging - 1.0.0-beta.19424.1
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19424.1
* Update dependencies from https://github.com/dotnet/arcade build
20190825.1
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19425.1
- Microsoft.DotNet.Build.Tasks.Feed - 2.2.0-beta.19425.1
- Microsoft.DotNet.Build.Tasks.Packaging - 1.0.0-beta.19425.1
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19425.1
* Update dependencies from https://github.com/dotnet/arcade build
20190826.12
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19426.12
- Microsoft.DotNet.Build.Tasks.Feed - 2.2.0-beta.19426.12
- Microsoft.DotNet.Build.Tasks.Packaging - 1.0.0-beta.19426.12
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19426.12
* Update dependencies from https://github.com/dotnet/arcade build
20190827.8
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19427.8
- Microsoft.DotNet.Build.Tasks.Feed - 2.2.0-beta.19427.8
- Microsoft.DotNet.Build.Tasks.Packaging - 1.0.0-beta.19427.8
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19427.8
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
- <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19421.4">
+ <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19427.8">
<Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>27a6926f6b1d4b36d86a236d2b05cdee3669589e</Sha>
+ <Sha>c02ca5d078b5b54bef8043d972082f41fd912190</Sha>
</Dependency>
- <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19421.4">
+ <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19427.8">
<Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>27a6926f6b1d4b36d86a236d2b05cdee3669589e</Sha>
+ <Sha>c02ca5d078b5b54bef8043d972082f41fd912190</Sha>
</Dependency>
- <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="2.2.0-beta.19421.4">
+ <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="2.2.0-beta.19427.8">
<Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>27a6926f6b1d4b36d86a236d2b05cdee3669589e</Sha>
+ <Sha>c02ca5d078b5b54bef8043d972082f41fd912190</Sha>
</Dependency>
- <Dependency Name="Microsoft.DotNet.Build.Tasks.Packaging" Version="1.0.0-beta.19421.4">
+ <Dependency Name="Microsoft.DotNet.Build.Tasks.Packaging" Version="1.0.0-beta.19427.8">
<Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>27a6926f6b1d4b36d86a236d2b05cdee3669589e</Sha>
+ <Sha>c02ca5d078b5b54bef8043d972082f41fd912190</Sha>
</Dependency>
<Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.6.0-rc1.19420.10">
<Uri>https://github.com/dotnet/corefx</Uri>
<UsingToolXliff>false</UsingToolXliff>
<!-- Package versions -->
<!-- arcade -->
- <MicrosoftDotNetBuildTasksFeedVersion>2.2.0-beta.19421.4</MicrosoftDotNetBuildTasksFeedVersion>
- <MicrosoftDotNetBuildTasksPackagingVersion>1.0.0-beta.19421.4</MicrosoftDotNetBuildTasksPackagingVersion>
+ <MicrosoftDotNetBuildTasksFeedVersion>2.2.0-beta.19427.8</MicrosoftDotNetBuildTasksFeedVersion>
+ <MicrosoftDotNetBuildTasksPackagingVersion>1.0.0-beta.19427.8</MicrosoftDotNetBuildTasksPackagingVersion>
<MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.19278.1</MicrosoftDotNetXUnitConsoleRunnerVersion>
<!-- roslyn -->
<MicrosoftNetCompilersToolsetVersion>3.3.0-beta2-19367-02</MicrosoftNetCompilersToolsetVersion>
}
else {
Write-Verbose "Downloading $Uri"
+ # Don't display the console progress UI - it's a huge perf hit
+ $ProgressPreference = 'SilentlyContinue'
while($Attempt -Lt $DownloadRetries)
{
try {
<CliArguments>--dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%</CliArguments>
<Python>py -3</Python>
<CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
+ <BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd</HelixPreCommands>
<ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+ <BaselineArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
+ <ResultsComparer>%HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj</ResultsComparer>
+ <DotnetExe>%HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe</DotnetExe>
+ <Percent>%25%25</Percent>
+ <XMLResults>%HELIX_WORKITEM_ROOT%\testResults.xml</XMLResults>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'false'">
<CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
<Python>python3</Python>
<CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
+ <BaselineCoreRun>$(BaseDirectory)/Baseline_Core_Root/corerun</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
<ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+ <BaselineArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
+ <ResultsComparer>$(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj</ResultsComparer>
+ <DotnetExe>$(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet</DotnetExe>
+ <Percent>%25</Percent>
+ <XMLResults>$HELIX_WORKITEM_ROOT/testResults.xml</XMLResults>
</PropertyGroup>
<PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
<CoreRunArgument>--corerun $(CoreRun)</CoreRunArgument>
</PropertyGroup>
+ <PropertyGroup Condition="'$(UseBaselineCoreRun)' == 'true'">
+ <BaselineCoreRunArgument>--corerun $(BaselineCoreRun)</BaselineCoreRunArgument>
+ </PropertyGroup>
+
<PropertyGroup Condition="'$(WorkItemCommand)' != ''">
<WorkItemCommand>$(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)</WorkItemCommand>
</PropertyGroup>
<Partition Include="$(BuildConfig).Partition4" Index="4" />
</ItemGroup>
+ <PropertyGroup Condition="'$(Compare)' == 'true'">
+ <FailOnTestFailure>false</FailOnTestFailure>
+ </PropertyGroup>
+
<!--
Partition the Microbenchmarks project, but nothing else
-->
<ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="@(Partition)">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+ <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --artifacts $(BaselineArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</PreCommands>
<Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+ <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)</PostCommands>
<Timeout>4:00</Timeout>
</HelixWorkItem>
</ItemGroup>
+
<ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="$(BuildConfig).WorkItem">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+ <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --artifacts $(ArtifactsDirectory)"</PreCommands>
<Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)"</Command>
+ <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)</PostCommands>
<Timeout>4:00</Timeout>
</HelixWorkItem>
</ItemGroup>
Param(
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
[string] $CoreRootDirectory,
+ [string] $BaselineCoreRootDirectory,
[string] $Architecture="x64",
[string] $Framework="netcoreapp5.0",
[string] $CompilationMode="Tiered",
[string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
[string] $Kind="micro",
[switch] $Internal,
+ [switch] $Compare,
[string] $Configurations="CompilationMode=$CompilationMode"
)
$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance")
$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
+$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty)
$PayloadDirectory = (Join-Path $SourceDirectory "Payload")
$PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
if ($Framework.StartsWith("netcoreapp")) {
- $Queue = "Windows.10.Amd64.ClientRS4.Open"
+ $Queue = "Windows.10.Amd64.ClientRS5.Open"
+}
+
+if ($Compare) {
+ $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
+ $PerfLabArguments = ""
+ $ExtraBenchmarkDotNetArguments = ""
}
if ($Internal) {
$NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
}
+if ($UseBaselineCoreRun) {
+ $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root")
+ Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
+}
$DocsDir = (Join-Path $PerformanceDirectory "docs")
robocopy $DocsDir $WorkItemDirectory
Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false
# Helix Arguments
Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false
source_directory=$BUILD_SOURCESDIRECTORY
core_root_directory=
+baseline_core_root_directory=
architecture=x64
framework=netcoreapp5.0
compilation_mode=tiered
commit_sha=$BUILD_SOURCEVERSION
build_number=$BUILD_BUILDNUMBER
internal=false
+compare=false
kind="micro"
run_categories="coreclr corefx"
csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
configurations=
run_from_perf_repo=false
use_core_run=true
+use_baseline_core_run=true
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
core_root_directory=$2
shift 2
;;
+ --baselinecorerootdirectory)
+ baseline_core_root_directory=$2
+ shift 2
+ ;;
--architecture)
architecture=$2
shift 2
internal=true
shift 1
;;
+ --compare)
+ compare=true
+ shift 1
+ ;;
--configurations)
configurations=$2
shift 2
use_core_run=false
fi
+if [ -z "$baseline_core_root_directory" ]; then
+ use_baseline_core_run=false
+fi
+
payload_directory=$source_directory/Payload
performance_directory=$payload_directory/performance
workitem_directory=$source_directory/workitem
creator=$BUILD_DEFINITIONNAME
helix_source_prefix="pr"
+if [[ "$compare" == true ]]; then
+ extra_benchmark_dotnet_arguments=
+ perflab_arguments=
+
+ # No open queues for arm64
+ if [[ "$architecture" = "arm64" ]]; then
+ echo "Compare not available for arm64"
+ exit 1
+ fi
+
+ queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
+fi
+
if [[ "$internal" == true ]]; then
perflab_arguments="--upload-to-perflab-container"
helix_source_prefix="official"
mv $core_root_directory $new_core_root
fi
+if [[ "$use_baseline_core_run" = true ]]; then
+ new_baseline_core_root=$payload_directory/Baseline_Core_Root
+ mv $baseline_core_root_directory $new_baseline_core_root
+fi
+
ci=true
_script_dir=$(pwd)/eng/common
# Make sure all of our variables are available for future steps
Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
+Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false
Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
-Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
\ No newline at end of file
+Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false
}
$ValidationFailures = 0
- foreach ($Job in $Jobs) {
+ foreach ($Job in @(Get-Job)) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job
if ($jobResult -ne "0") {
$ValidationFailures++
Set-StrictMode -Version 2.0
$LASTEXITCODE = 0
+# Don't display the console progress UI - it's a huge perf hit
+$ProgressPreference = 'SilentlyContinue'
+
# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
$installScript = Join-Path $dotnetRoot "dotnet-install.ps1"
if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript
}
Create-Directory $packageDir
Write-Host "Downloading $packageName $packageVersion"
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath
Unzip $packagePath $packageDir
}
# Use curl if available, otherwise use wget
if command -v curl > /dev/null; then
- curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script"
- else
- wget -q -O "$install_script" "$install_script_url"
+ curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ else
+ wget -q -O "$install_script" "$install_script_url" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
fi
fi
-
# return value
_GetDotNetInstallScript="$install_script"
}
"python": "2.7.15"
},
"msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19421.4",
- "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19421.4",
+ "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19427.8",
+ "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19427.8",
"Microsoft.Build.NoTargets": "1.0.53",
"Microsoft.Build.Traversal": "2.0.2"
}