Run IBCMerge step for Linux on Windows (dotnet/coreclr#20496)

This change does the following:

* Move the IBCOptimize step out of the Crossgen section and into the CoreLib build section of build.cmd
* Adds -ibconly which will skip building System.Private.CoreLib and only run the ibcmerge step
* Adds crossgenonly and partialngen flags to build.sh

These three changes facilitate our ability to apply IBC data to Linux assemblies on Windows and the perform the crossgen step on Linux, which will be our flow for official builds when we want to apply IBC data since IBCMerge cannot run on non-Windows platforms

Commit migrated from https://github.com/dotnet/coreclr/commit/ebc4bde180020a252e681cec44956fda18afdb8c

diff --git a/src/coreclr/build.cmd b/src/coreclr/build.cmd
index 3541899..75afd3e 100644 (file)
--- a/src/coreclr/build.cmd
+++ b/src/coreclr/build.cmd
@@ -75,6 +75,7 @@ set __PgoOptimize=1
 set __EnforcePgo=0
 set __IbcTuning=
 set __IbcOptimize=0
+set __IbcOnly=0
 
 REM __PassThroughArgs is a set of things that will be passed through to nested calls to build.cmd
 REM when using "all".
@@ -182,6 +183,7 @@ if /i "%1" == "-pgoinstrument"       (set __PgoInstrument=1&set processedArgs=!p
 if /i "%1" == "-enforcepgo"          (set __EnforcePgo=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "-nopgooptimize"       (set __PgoOptimize=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "-ibcoptimize"         (set __IbcOptimize=1&set __PartialNgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "-ibconly"             (set __IbcOptimize=1&set __PartialNgen=1&set __IbcOnly=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "-ibcinstrument"       (set __IbcTuning=/Tuning&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "-crossgenaltjit"      (set __CrossgenAltJit=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
 
@@ -579,104 +581,48 @@ if %__BuildCoreLib% EQU 1 (
     REM Scope environment changes start {
     setlocal
 
-    echo %__MsgPrefix%Commencing build of System.Private.CoreLib for %__BuildOS%.%__BuildArch%.%__BuildType%
-    rem Explicitly set Platform causes conflicts in CoreLib project files. Clear it to allow building from VS x64 Native Tools Command Prompt
-    set Platform=
+    if %__IbcOnly% EQU 0 (
+        echo %__MsgPrefix%Commencing build of System.Private.CoreLib for %__BuildOS%.%__BuildArch%.%__BuildType%
+        rem Explicitly set Platform causes conflicts in CoreLib project files. Clear it to allow building from VS x64 Native Tools Command Prompt
+        set Platform=
 
-    set __ExtraBuildArgs=
+        set __ExtraBuildArgs=
 
-    if "%__BuildSOS%" == "0" (
-        set __ExtraBuildArgs=!__ExtraBuildArgs! -SkipSOS=true
-    )
-
-    if "%__BuildManagedTools%" == "1" (
-        set __ExtraBuildArgs=!__ExtraBuildArgs! -BuildManagedTools=true
-    )
-
-    if "%__SkipNugetPackage%" == "1" (
-        set __ExtraBuildArgs=!__ExtraBuildArgs! -buildNugetPackage=false
-    ) else (
-        set __ExtraBuildArgs=!__ExtraBuildArgs! -buildNugetPackage=true
-    )
-
-    set __BuildLogRootName=System.Private.CoreLib
-    set __BuildLog="%__LogsDir%\!__BuildLogRootName!_%__BuildOS%__%__BuildArch%__%__BuildType%.log"
-    set __BuildWrn="%__LogsDir%\!__BuildLogRootName!_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn"
-    set __BuildErr="%__LogsDir%\!__BuildLogRootName!_%__BuildOS%__%__BuildArch%__%__BuildType%.err"
-    set __MsbuildLog=/flp:Verbosity=normal;LogFile=!__BuildLog!
-    set __MsbuildWrn=/flp1:WarningsOnly;LogFile=!__BuildWrn!
-    set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr!
-    set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr!
-
-    call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\build.proj !__Logging! %__RunArgs% !__ExtraBuildArgs! %__UnprocessedBuildArgs%
-
-    if not !errorlevel! == 0 (
-        echo %__MsgPrefix%Error: System.Private.CoreLib build failed. Refer to the build log files for details:
-        echo     !__BuildLog!
-        echo     !__BuildWrn!
-        echo     !__BuildErr!
-        exit /b 1
-    )
-
-    REM } Scope environment changes end
-    endlocal
-)
-
-REM =========================================================================================
-REM ===
-REM === Build native System.Private.CoreLib.
-REM ===
-REM =========================================================================================
-
-REM Scope environment changes start {
-setlocal
-
-REM Need diasymreader.dll on your path for /CreatePdb
-set PATH=%PATH%;%WinDir%\Microsoft.Net\Framework64\V4.0.30319;%WinDir%\Microsoft.Net\Framework\V4.0.30319
-
-if %__BuildNativeCoreLib% EQU 1 (
-    echo %__MsgPrefix%Generating native image of System.Private.CoreLib for %__BuildOS%.%__BuildArch%.%__BuildType%. Logging to "%__CrossGenCoreLibLog%".
-    if exist "%__CrossGenCoreLibLog%" del "%__CrossGenCoreLibLog%"
-
-    REM Need VS native tools environment for the **target** arch when running instrumented binaries
-    if %__PgoInstrument% EQU 1 (
-        set __VCExecArch=%__BuildArch%
-        if /i [%__BuildArch%] == [x64] set __VCExecArch=amd64
-        echo %__MsgPrefix%Using environment: "%__VCToolsRoot%\vcvarsall.bat" !__VCExecArch!
-        call                                 "%__VCToolsRoot%\vcvarsall.bat" !__VCExecArch!
-        @if defined _echo @echo on
-        if NOT !errorlevel! == 0 (
-            echo %__MsgPrefix%Error: Failed to load native tools environment for !__VCExecArch!
-            goto CrossgenFailure
+        if "%__BuildSOS%" == "0" (
+            set __ExtraBuildArgs=!__ExtraBuildArgs! -SkipSOS=true
         )
 
-        REM HACK: Workaround for [dotnet/coreclr#13970](https://github.com/dotnet/coreclr/issues/13970)
-        set __PgoRtPath=
-        for /f "tokens=*" %%f in ('where pgort*.dll') do (
-            if not defined __PgoRtPath set "__PgoRtPath=%%~f"
+        if "%__BuildManagedTools%" == "1" (
+            set __ExtraBuildArgs=!__ExtraBuildArgs! -BuildManagedTools=true
         )
-        echo %__MsgPrefix%Copying "!__PgoRtPath!" into "%__BinDir%"
-        copy /y "!__PgoRtPath!" "%__BinDir%" || (
-            echo %__MsgPrefix%Error: copy failed
-            goto CrossgenFailure
+
+        if "%__SkipNugetPackage%" == "1" (
+            set __ExtraBuildArgs=!__ExtraBuildArgs! -buildNugetPackage=false
+        ) else (
+            set __ExtraBuildArgs=!__ExtraBuildArgs! -buildNugetPackage=true
         )
-        REM End HACK
-    )
 
-    if defined __CrossgenAltJit (
-        REM Set altjit flags for the crossgen run. Note that this entire crossgen section is within a setlocal/endlocal scope,
-        REM so we don't need to save or unset these afterwards.
-        echo %__MsgPrefix%Setting altjit environment variables for %__CrossgenAltJit%.
-        echo %__MsgPrefix%Setting altjit environment variables for %__CrossgenAltJit%. >> "%__CrossGenCoreLibLog%"
-        set COMPlus_AltJit=*
-        set COMPlus_AltJitNgen=*
-        set COMPlus_AltJitName=%__CrossgenAltJit%
-        set COMPlus_AltJitAssertOnNYI=1
-        set COMPlus_NoGuiOnAssert=1
-        set COMPlus_ContinueOnAssert=0
+        set __BuildLogRootName=System.Private.CoreLib
+        set __BuildLog="%__LogsDir%\!__BuildLogRootName!_%__BuildOS%__%__BuildArch%__%__BuildType%.log"
+        set __BuildWrn="%__LogsDir%\!__BuildLogRootName!_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn"
+        set __BuildErr="%__LogsDir%\!__BuildLogRootName!_%__BuildOS%__%__BuildArch%__%__BuildType%.err"
+        set __MsbuildLog=/flp:Verbosity=normal;LogFile=!__BuildLog!
+        set __MsbuildWrn=/flp1:WarningsOnly;LogFile=!__BuildWrn!
+        set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr!
+        set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr!
+
+        call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\build.proj !__Logging! %__RunArgs% !__ExtraBuildArgs! %__UnprocessedBuildArgs%
+
+        if not !errorlevel! == 0 (
+            echo %__MsgPrefix%Error: System.Private.CoreLib build failed. Refer to the build log files for details:
+            echo     !__BuildLog!
+            echo     !__BuildWrn!
+            echo     !__BuildErr!
+            exit /b 1
+        )
     )
-
     if %__IbcOptimize% EQU 1 (
+        echo %__MsgPrefix%Commencing IBCMerge of System.Private.CoreLib for %__BuildOS%.%__BuildArch%.%__BuildType%
         set IbcMergeProjectFilePath=%__ProjectDir%\src\.nuget\optdata\ibcmerge.csproj
         for /f "tokens=*" %%s in ('%DotNetCli% msbuild "!IbcMergeProjectFilePath!" /t:DumpIbcMergePackageVersion /nologo') do @(
             set __IbcMergeVersion=%%s
@@ -697,7 +643,7 @@ if %__BuildNativeCoreLib% EQU 1 (
                 echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"
                 !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1
                 if NOT !errorlevel! == 0 (
-                    echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                    echo %__MsgPrefix%Error: IbcMerge of System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
                     REM Put it in the same log, helpful for Jenkins
                     type %__CrossGenCoreLibLog%
                     goto CrossgenFailure
@@ -708,7 +654,7 @@ if %__BuildNativeCoreLib% EQU 1 (
                 echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"
                 !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1
                 if NOT !errorlevel! == 0 (
-                    echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                    echo %__MsgPrefix%Error: IbcMerge of System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
                     REM Put it in the same log, helpful for Jenkins
                     type %__CrossGenCoreLibLog%
                     goto CrossgenFailure
@@ -727,7 +673,7 @@ if %__BuildNativeCoreLib% EQU 1 (
                 echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"
                 !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1
                 if NOT !errorlevel! == 0 (
-                    echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                    echo %__MsgPrefix%Error: IbcMerge of System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
                     REM Put it in the same log, helpful for Jenkins
                     type %__CrossGenCoreLibLog%
                     goto CrossgenFailure
@@ -738,14 +684,14 @@ if %__BuildNativeCoreLib% EQU 1 (
                 echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%"
                 !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1
                 if NOT !errorlevel! == 0 (
-                    echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                    echo %__MsgPrefix%Error: IbcMerge of System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
                     REM Put it in the same log, helpful for Jenkins
                     type %__CrossGenCoreLibLog%
                     goto CrossgenFailure
                 )
             ) else (
                 echo %__MsgPrefix%!TargetOptimizationDataFile! does not exist >> %__CrossGenCoreLibLog%
-                echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
+                echo %__MsgPrefix%Error: IbcMerge of System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog%
                 REM Put it in the same log, helpful for Jenkins
                 type %__CrossGenCoreLibLog%
                 goto CrossgenFailure
@@ -753,6 +699,64 @@ if %__BuildNativeCoreLib% EQU 1 (
         )
     )
 
+    REM } Scope environment changes end
+    endlocal
+)
+
+REM =========================================================================================
+REM ===
+REM === Build native System.Private.CoreLib.
+REM ===
+REM =========================================================================================
+
+REM Scope environment changes start {
+setlocal
+
+REM Need diasymreader.dll on your path for /CreatePdb
+set PATH=%PATH%;%WinDir%\Microsoft.Net\Framework64\V4.0.30319;%WinDir%\Microsoft.Net\Framework\V4.0.30319
+
+if %__BuildNativeCoreLib% EQU 1 (
+    echo %__MsgPrefix%Generating native image of System.Private.CoreLib for %__BuildOS%.%__BuildArch%.%__BuildType%. Logging to "%__CrossGenCoreLibLog%".
+    if exist "%__CrossGenCoreLibLog%" del "%__CrossGenCoreLibLog%"
+
+    REM Need VS native tools environment for the **target** arch when running instrumented binaries
+    if %__PgoInstrument% EQU 1 (
+        set __VCExecArch=%__BuildArch%
+        if /i [%__BuildArch%] == [x64] set __VCExecArch=amd64
+        echo %__MsgPrefix%Using environment: "%__VCToolsRoot%\vcvarsall.bat" !__VCExecArch!
+        call                                 "%__VCToolsRoot%\vcvarsall.bat" !__VCExecArch!
+        @if defined _echo @echo on
+        if NOT !errorlevel! == 0 (
+            echo %__MsgPrefix%Error: Failed to load native tools environment for !__VCExecArch!
+            goto CrossgenFailure
+        )
+
+        REM HACK: Workaround for [dotnet/coreclr#13970](https://github.com/dotnet/coreclr/issues/13970)
+        set __PgoRtPath=
+        for /f "tokens=*" %%f in ('where pgort*.dll') do (
+            if not defined __PgoRtPath set "__PgoRtPath=%%~f"
+        )
+        echo %__MsgPrefix%Copying "!__PgoRtPath!" into "%__BinDir%"
+        copy /y "!__PgoRtPath!" "%__BinDir%" || (
+            echo %__MsgPrefix%Error: copy failed
+            goto CrossgenFailure
+        )
+        REM End HACK
+    )
+
+    if defined __CrossgenAltJit (
+        REM Set altjit flags for the crossgen run. Note that this entire crossgen section is within a setlocal/endlocal scope,
+        REM so we don't need to save or unset these afterwards.
+        echo %__MsgPrefix%Setting altjit environment variables for %__CrossgenAltJit%.
+        echo %__MsgPrefix%Setting altjit environment variables for %__CrossgenAltJit%. >> "%__CrossGenCoreLibLog%"
+        set COMPlus_AltJit=*
+        set COMPlus_AltJitNgen=*
+        set COMPlus_AltJitName=%__CrossgenAltJit%
+        set COMPlus_AltJitAssertOnNYI=1
+        set COMPlus_NoGuiOnAssert=1
+        set COMPlus_ContinueOnAssert=0
+    )
+
     if defined __PartialNgen (
         set COMPlus_PartialNgen=1
     )
@@ -965,6 +969,8 @@ echo -nopgooptimize: do not use profile guided optimizations.
 echo -enforcepgo: verify after the build that PGO was used for key DLLs, and fail the build if not
 echo -pgoinstrument: generate instrumented code for profile guided optimization enabled binaries.
 echo -ibcinstrument: generate IBC-tuning-enabled native images when invoking crossgen.
+echo -ibcoptimize: use IBC data to optimize System.Private.CoreLib.dll
+echo -ibconly: only run the ibcoptimize step. Assumes an appropriate build already exists
 echo -configureonly: skip all builds; only run CMake ^(default: CMake and builds are run^)
 echo -skipconfigure: skip CMake ^(default: CMake is run^)
 echo -skipmscorlib: skip building System.Private.CoreLib ^(default: System.Private.CoreLib is built^).

diff --git a/src/coreclr/build.sh b/src/coreclr/build.sh
index 914953f..31994c9 100755 (executable)
--- a/src/coreclr/build.sh
+++ b/src/coreclr/build.sh
@@ -48,6 +48,8 @@ usage()
     echo "-skipnuget - skip building nuget packages."
     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
     echo "-skipcrossgen - skip native image generation"
+    echo "-crossgenonly - only run native image generation"
+    echo "-partialngen - build CoreLib as PartialNGen"
     echo "-verbose - optional argument to enable verbose build output."
     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
@@ -428,6 +430,10 @@ build_CoreLib_ni()
 {
     local __CrossGenExec=$1
 
+    if [ $__PartialNgen == 1 ]; then
+        export COMPlus_PartialNGen=1
+    fi
+
     if [ -e $__CrossGenCoreLibLog ]; then
         rm $__CrossGenCoreLibLog
     fi
@@ -523,7 +529,7 @@ generate_NugetPackages()
     fi
 
     # Since we can build mscorlib for this OS, did we build the native components as well?
-    if [ $__SkipCoreCLR == 1 ]; then
+    if [[ $__SkipCoreCLR == 1 && $__CrossgenOnly == 0 ]]; then
         echo "Unable to generate nuget packages since native components were not built."
         return
     fi
@@ -659,6 +665,8 @@ __SkipCoreCLR=0
 __SkipMSCorLib=0
 __SkipRestoreOptData=0
 __SkipCrossgen=0
+__CrossgenOnly=0
+__PartialNgen=0
 __SkipTests=0
 __CrossBuild=0
 __ClangMajorVersion=0
@@ -858,6 +866,15 @@ while :; do
             __SkipCrossgen=1
             ;;
 
+        crossgenonly|-crossgenonly)
+            __SkipMSCorLib=1
+            __SkipCoreCLR=1
+            __CrossgenOnly=1
+            ;;
+        partialngen|-partialngen)
+            __PartialNgen=1
+            ;;
+
         skiptests|-skiptests)
             __SkipTests=1
             ;;
@@ -1045,6 +1062,10 @@ fi
 
 build_CoreLib
 
+if [ $__CrossgenOnly ==1 ]; then
+    build_CoreLib_ni "$__BinDir/crossgen"
+fi
+
 # Generate nuget packages
 if [ $__SkipNuget != 1 ]; then
     generate_NugetPackages