Add IBC support (#19046)

This change adds support to consume IBC data packages that are created by the dotnet/optimization repository. With these changes and dotnet/buildtools#2103 to enable IBC optimizations for corefx, we will see a 32% decrease in the size of NetCoreApp (and an overall decrease of 16%), a 30% reduction in ref set, a 5% improvement in time to first request and steady state performance for MusicStore.

Size

 	Crossgen	Partial	Partial vs Crossgen (lower is better)
Shared (MB)	185.6	137.3	0.74
Microsoft.NETCore.App (MB)	115.6	67.4	0.58
RefSet

MusicStore

 	Crossgen	Partial	Partial vs Crossgen (lower is better)
Total	19.389	13.472	0.69
AllReady

Crossgen	Partial	Partial vs Crossgen (lower is better)
Total	17.58	12.214
Performance

MusicStore

 	Crossgen	Partial	Partial vs Crossgen (lower is better)
Server Start (ms)	870	870.6	1.00
First request (ms)	3532.6	3386.6	0.95
Steady State (ms)	2.926	2.79	0.95
AllReady

Crossgen	Partial	Partial vs Crossgen (lower is better)
Server Start (ms)	2102	1942.4
First Request (ms)	4263.2	4126
Steady State (ms)	5.69	5.68
TechEmpower Plaintext

Crossgen	Partial	Partial vs Crossgen (lower is better)
Requests per Second	1928649.8	1893183.8
First Request (ms)	76.14	80.11
Startup (ms)	391	372.8
Working Set (MB)	382.2	373.4

diff --git a/build.cmd b/build.cmd
index 2100d24..5bee1b8 100644 (file)
--- a/build.cmd
+++ b/build.cmd
@@ -74,6 +74,7 @@ set __PgoInstrument=0
 set __PgoOptimize=1
 set __EnforcePgo=0
 set __IbcTuning=
+set __IbcOptimize=0
 
 REM __PassThroughArgs is a set of things that will be passed through to nested calls to build.cmd
 REM when using "all".
@@ -100,6 +101,7 @@ set __CrossArch=
 set __SkipNugetPackage=0
 set __PgoOptDataVersion=
 set __IbcOptDataVersion=
+set __IbcMergeVersion=
 
 @REM CMD has a nasty habit of eating "=" on the argument list, so passing:
 @REM    -priority=1
@@ -178,6 +180,7 @@ if /i "%1" == "-usenmakemakefiles"   (set __NMakeMakefiles=1&set __ConfigureOnly
 if /i "%1" == "-pgoinstrument"       (set __PgoInstrument=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "-enforcepgo"          (set __EnforcePgo=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "-nopgooptimize"       (set __PgoOptimize=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "-ibcoptimize"         (set __IbcOptimize=1&set __PartialNgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "-ibcinstrument"       (set __IbcTuning=/Tuning&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "-crossgenaltjit"      (set __CrossgenAltJit=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
 
@@ -580,10 +583,6 @@ if %__BuildCoreLib% EQU 1 (
     set Platform=
 
     set __ExtraBuildArgs=
-    if not defined __IbcTuning (
-        set __ExtraBuildArgs=!__ExtraBuildArgs! -OptimizationDataDir="%__PackagesDir%/optimization.%__BuildOS%-%__BuildArch%.IBC.CoreCLR/%__IbcOptDataVersion%/data/"
-        set __ExtraBuildArgs=!__ExtraBuildArgs! -EnableProfileGuidedOptimization=true
-    )
 
     if "%__BuildSOS%" == "0" (
         set __ExtraBuildArgs=!__ExtraBuildArgs! -SkipSOS=true
@@ -676,6 +675,87 @@ if %__BuildNativeCoreLib% EQU 1 (
         set COMPlus_ContinueOnAssert=0
     )
 
+    if %__IbcOptimize% EQU 1 (
+        set IbcMergeProjectFilePath=%__ProjectDir%\src\.nuget\optdata\ibcmerge.csproj
+        for /f "tokens=*" %%s in ('%DotNetCli% msbuild "!IbcMergeProjectFilePath!" /t:DumpIbcMergePackageVersion /nologo') do @(
+            set __IbcMergeVersion=%%s
+        )
+
+        set IbcMergePath=%__PackagesDir%\microsoft.dotnet.ibcmerge\!__IbcMergeVersion!\lib\net45\ibcmerge.exe
+        if exist !IbcMergePath! (
+            echo %__MsgPrefix%Optimizing using IBC training data
+            set OptimizationDataDir=%__PackagesDir%\optimization.%__BuildOS%-%__BuildArch%.IBC.CoreCLR\!__IbcOptDataVersion!\data\
+            set InputAssemblyFile=!OptimizationDataDir!System.Private.CoreLib.dll
+            set TargetOptimizationDataFile=!OptimizationDataDir!System.Private.CoreLib.pgo
+
+            if exist "!InputAssemblyFile!" (
+                set RawOptimizationDataFile=!OptimizationDataDir!System.Private.CoreLib.ibc
+
+                REM Merge the optimization data into the source DLL
+                set NEXTCMD="!IbcMergePath!" -q -f -delete -mo "!InputAssemblyFile!" "!RawOptimizationDataFile!"
+                echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"
+                !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1
+                if NOT !errorlevel! == 0 (
+                    echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                    REM Put it in the same log, helpful for Jenkins
+                    type %__CrossGenCoreLibLog%
+                    goto CrossgenFailure
+                )
+
+                REM Verify that the optimization data has been merged
+                set NEXTCMD="!IbcMergePath!" -mi "!InputAssemblyFile!"
+                echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"
+                !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1
+                if NOT !errorlevel! == 0 (
+                    echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                    REM Put it in the same log, helpful for Jenkins
+                    type %__CrossGenCoreLibLog%
+                    goto CrossgenFailure
+                )
+
+                REM Save the module as *.pgo to match the convention expected
+                copy /y !InputAssemblyFile! !TargetOptimizationDataFile!
+            )
+
+            if exist "!TargetOptimizationDataFile!" (
+                REM Customize IBCMerge's arguments depending on input props
+                set IBCMergeArguments=-q -f -delete -mo "%__BinDir%\IL\System.Private.CoreLib.dll" -incremental "!TargetOptimizationDataFile!"
+
+                REM Apply optimization data to the compiled assembly
+                set NEXTCMD="!IbcMergePath!" !IBCMergeArguments!
+                echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"
+                !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1
+                if NOT !errorlevel! == 0 (
+                    echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                    REM Put it in the same log, helpful for Jenkins
+                    type %__CrossGenCoreLibLog%
+                    goto CrossgenFailure
+                )
+                
+                REM Verify that the optimization data has been applied
+                set NEXTCMD="!IbcMergePath!" -mi "%__BinDir%\IL\System.Private.CoreLib.dll"
+                echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"
+                !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1
+                if NOT !errorlevel! == 0 (
+                    echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                    REM Put it in the same log, helpful for Jenkins
+                    type %__CrossGenCoreLibLog%
+                    goto CrossgenFailure
+                )
+            ) else (
+                echo %__MsgPrefix%!TargetOptimizationDataFile! does not exist >> %__CrossGenCoreLibLog%
+                echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                REM Put it in the same log, helpful for Jenkins
+                type %__CrossGenCoreLibLog%
+                goto CrossgenFailure
+            )
+        )
+    )
+
+    if defined __PartialNgen (
+        set COMPlus_PartialNgen=1
+    )
+
     set NEXTCMD="%__CrossgenExe%" %__IbcTuning% /Platform_Assemblies_Paths "%__BinDir%"\IL /out "%__BinDir%\System.Private.CoreLib.dll" "%__BinDir%\IL\System.Private.CoreLib.dll"
     echo %__MsgPrefix%!NEXTCMD!
     echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"

diff --git a/build.sh b/build.sh
index d66c9c6..95e21b5 100755 (executable)
--- a/build.sh
+++ b/build.sh
@@ -193,7 +193,7 @@ restore_optdata()
         fi
         local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
         __PgoOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
-        __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^\s*//')
+        __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^[[:blank:]]*//')
     fi
 }
 

diff --git a/buildpipeline/DotNet-CoreClr-Trusted-Windows.json b/buildpipeline/DotNet-CoreClr-Trusted-Windows.json
index 1011587..f550430 100644 (file)
--- a/buildpipeline/DotNet-CoreClr-Trusted-Windows.json
+++ b/buildpipeline/DotNet-CoreClr-Trusted-Windows.json
@@ -102,7 +102,7 @@
       },
       "inputs": {
         "filename": "sync.cmd",
-        "arguments": "-p -- /p:BuildType=$(PB_BuildType)",
+        "arguments": "-p -- /p:BuildType=$(PB_BuildType) $(PB_OptionalToolingSyncArguments)",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -122,7 +122,7 @@
       },
       "inputs": {
         "filename": "build.cmd",
-        "arguments": "$(Architecture) $(PB_BuildType) skiptests skipbuildpackages $(PB_EnforcePGO) -OfficialBuildId=$(OfficialBuildId) -Priority=$(Priority) -skiprestore -- /p:SignType=$(PB_SignType) /flp:\"v=diag\"",
+        "arguments": "$(Architecture) $(PB_BuildType) skiptests skipbuildpackages -ibcoptimize $(PB_EnforcePGO) -OfficialBuildId=$(OfficialBuildId) -Priority=$(Priority) -skiprestore -- /p:SignType=$(PB_SignType) /flp:\"v=diag\"",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -386,6 +386,24 @@
     "Label": {
       "value": "$(Build.BuildNumber)"
     },
+    "PB_OptionalToolingSyncArguments": {
+      "value": "/p:OptionalToolSource=$(PB_OptionalToolSource) /p:OptionalToolSourceUser=$(PB_OptionalToolSourceUser);OptionalToolSourcePassword=$(PB_OptionalToolSourcePAT)"
+    },
+    "PB_OptionalToolSource": {
+      "value": null,
+      "allowOverride": true,
+      "isSecret": true
+    },
+    "PB_OptionalToolSourcePAT": {
+      "value": null,
+      "allowOverride": true,
+      "isSecret": true
+    },
+    "PB_OptionalToolSourceUser": {
+      "value": null,
+      "allowOverride": true,
+      "isSecret": true
+    },
     "PB_SignType": {
       "value": "test",
       "allowOverride": true

diff --git a/dependencies.props b/dependencies.props
index 0c3724f..9b23470 100644 (file)
--- a/dependencies.props
+++ b/dependencies.props
@@ -28,6 +28,7 @@
     <BuildToolsCurrentRef>e0c1d36d8a19fed21cadfff4890e027bd4c2c214</BuildToolsCurrentRef>
     <PgoDataCurrentRef>e0c1d36d8a19fed21cadfff4890e027bd4c2c214</PgoDataCurrentRef>
     <CoreSetupCurrentRef>e0c1d36d8a19fed21cadfff4890e027bd4c2c214</CoreSetupCurrentRef>
+    <IbcDataCurrentRef>e0c1d36d8a19fed21cadfff4890e027bd4c2c214</IbcDataCurrentRef>
   </PropertyGroup>
 
   <!-- Tests/infrastructure dependency versions. -->
@@ -38,6 +39,8 @@
     <MicrosoftNETCoreRuntimeCoreCLRPackageVersion>3.0.0-preview1-26817-05</MicrosoftNETCoreRuntimeCoreCLRPackageVersion>
     <MicrosoftNETCoreAppPackageVersion>3.0.0-preview1-26817-01</MicrosoftNETCoreAppPackageVersion>
     <XunitPackageVersion>2.4.0-beta.2.build4010</XunitPackageVersion>
+    <IbcDataPackageVersion>99.99.99-master-20180817-0151</IbcDataPackageVersion>
+    <IbcMergePackageVersion>4.6.0-alpha-00001</IbcMergePackageVersion>
     <XunitPerformanceApiPackageVersion>1.0.0-beta-build0015</XunitPerformanceApiPackageVersion>
     <MicrosoftDiagnosticsTracingTraceEventPackageVersion>2.0.4</MicrosoftDiagnosticsTracingTraceEventPackageVersion>
     <CommandLineParserVersion>2.2.0</CommandLineParserVersion>
@@ -91,6 +94,10 @@
       <BuildInfoPath>$(BaseDotNetBuildInfo)optimization/$(DependencyBranch)</BuildInfoPath>
       <CurrentRef>$(PgoDataCurrentRef)</CurrentRef>
     </RemoteDependencyBuildInfo>
+    <RemoteDependencyBuildInfo Include="IbcData">
+      <BuildInfoPath>$(BaseDotNetBuildInfo)optimization/$(DependencyBranch)</BuildInfoPath>
+      <CurrentRef>$(IbcDataCurrentRef)</CurrentRef>
+    </RemoteDependencyBuildInfo>
     <RemoteDependencyBuildInfo Include="CoreSetup">
       <BuildInfoPath>$(BaseDotNetBuildInfo)core-setup/$(DependencyBranch)</BuildInfoPath>
       <CurrentRef>$(CoreSetupCurrentRef)</CurrentRef>
@@ -135,6 +142,11 @@
       <ElementName>PgoDataPackageVersion</ElementName>
       <PackageId>optimization.PGO.CoreCLR</PackageId>
     </XmlUpdateStep>
+    <XmlUpdateStep Include="Data">
+      <Path>$(MSBuildThisFileFullPath)</Path>
+      <ElementName>IbcDataPackageVersion</ElementName>
+      <PackageId>optimization.IBC.CoreCLR</PackageId>
+    </XmlUpdateStep>
     <UpdateStep Include="ILAsm">
       <UpdaterType>File</UpdaterType>
       <Path>$(MSBuildThisFileDirectory)ILAsmVersion.txt</Path>

diff --git a/src/.nuget/optdata/ibcmerge.csproj b/src/.nuget/optdata/ibcmerge.csproj
new file mode 100644 (file)
index 0000000..e69ac17
--- /dev/null
+++ b/src/.nuget/optdata/ibcmerge.csproj
@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+
+  <PropertyGroup>
+    <TargetFramework>netstandard</TargetFramework>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <RuntimeIdentifiers>win7-x64;win7-x86;linux-x64</RuntimeIdentifiers>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="microsoft.dotnet.ibcmerge" Version="$(IbcMergePackageVersion)" Condition="'$(IbcMergePackageVersion)'!=''" />
+  </ItemGroup>
+
+  <Target Name="DumpIbcMergePackageVersion">
+    <Message Importance="high" Text="$(IbcMergePackageVersion)" />
+  </Target>
+
+</Project>